From cb544c3fcca3708ca9882569e9135e3d2c6c25fc Mon Sep 17 00:00:00 2001 From: GGRei Date: Mon, 27 Apr 2026 17:54:26 +0200 Subject: [PATCH 1/3] dtm2: introduce parsed runtime template manager --- vlib/x/templating/dtm/README.md | 9 + .../templating/dtm/dynamic_template_manager.v | 926 +++++++++--------- .../dynamic_template_manager_behavior_test.v | 333 +++++++ ...namic_template_manager_cache_system_test.v | 64 +- .../dynamic_template_manager_dtm2_bridge.v | 65 ++ .../dtm/dynamic_template_manager_test.v | 2 +- .../dtm/escape_html_strings_in_templates.v | 27 +- vlib/x/templating/dtm/tmpl.v | 24 +- vlib/x/templating/dtm2/README.md | 365 +++++++ .../dtm2/benchmarks/dtm2_benchmark.v | 212 ++++ .../dtm2/benchmarks/run_dtm2_benchmark.sh | 57 ++ vlib/x/templating/dtm2/dtm2_extensions.json | 4 + .../dtm2/dynamic_template_manager2.v | 750 ++++++++++++++ .../dtm2/dynamic_template_manager2_test.v | 306 ++++++ 14 files changed, 2652 insertions(+), 492 deletions(-) create mode 100644 vlib/x/templating/dtm/dynamic_template_manager_behavior_test.v create mode 100644 vlib/x/templating/dtm/dynamic_template_manager_dtm2_bridge.v create mode 100644 vlib/x/templating/dtm2/README.md create mode 100644 vlib/x/templating/dtm2/benchmarks/dtm2_benchmark.v create mode 100755 vlib/x/templating/dtm2/benchmarks/run_dtm2_benchmark.sh create mode 100644 vlib/x/templating/dtm2/dtm2_extensions.json create mode 100644 vlib/x/templating/dtm2/dynamic_template_manager2.v create mode 100644 vlib/x/templating/dtm2/dynamic_template_manager2_test.v diff --git a/vlib/x/templating/dtm/README.md b/vlib/x/templating/dtm/README.md index af397e5a316c18..4034ed764ed315 100644 --- a/vlib/x/templating/dtm/README.md +++ b/vlib/x/templating/dtm/README.md @@ -3,6 +3,15 @@ A simple template manager integrated into the V project, designed to combine the power of V templates with Veb, without the need to recompile the application with every change. +> **Status:** `x.templating.dtm` is the legacy version of Dynamic Template +> Manager. New projects should use `x.templating.dtm2`, which provides the +> modern runtime renderer and parsed-template cache. +> +> Existing `dtm` users can keep their current API while migrating progressively: +> the v1 facade now delegates rendering to the DTM2 engine internally, preserving +> source compatibility while removing the old rendered-cache server from the +> main rendering path. + ## Quick Start Using the dynamic template manager (named `dtm` in this readme) is relatively straightforward. diff --git a/vlib/x/templating/dtm/dynamic_template_manager.v b/vlib/x/templating/dtm/dynamic_template_manager.v index 67505fd2f9c66a..4b0016af7c9bcc 100644 --- a/vlib/x/templating/dtm/dynamic_template_manager.v +++ b/vlib/x/templating/dtm/dynamic_template_manager.v @@ -1,32 +1,46 @@ +@[deprecated: 'use x.templating.dtm2 for new code'] +@[deprecated_after: '2999-01-01'] module dtm import os import crypto.md5 import hash.fnv1a import time -import regex +import strings +import x.templating.dtm2 // These are all the types of dynamic values that the DTM allows to be returned in the context of a map + +@[deprecated: 'use string placeholders with x.templating.dtm2.RenderParams for new code'] +@[deprecated_after: '2999-01-01'] pub type DtmMultiTypeMap = f32 | f64 | i16 | i64 | i8 | int | string | u16 | u32 | u64 | u8 // type MiddlewareFn = fn (mut Context, string) bool // cache_delay_expiration_at_min is the minimum setting for cache expiration delay, fixed at 5 minutes (measured in seconds). + +@[deprecated: 'use x.templating.dtm2 for new code'] +@[deprecated_after: '2999-01-01'] pub const cache_delay_expiration_at_min = 300 // cache_delay_expiration_at_max maximal is the maximal setting for cache expiration delay, fixed at 1 year (measured in seconds). + +@[deprecated: 'use x.templating.dtm2 for new code'] +@[deprecated_after: '2999-01-01'] pub const cache_delay_expiration_at_max = 31536000 // cache_delay_expiration_by_default is the default setting for cache expiration delay, fixed at 1 day (measured in seconds). + +@[deprecated: 'use x.templating.dtm2 for new code'] +@[deprecated_after: '2999-01-01'] pub const cache_delay_expiration_by_default = 86400 -// Setting channel capacity of the cache handler. +// Setting channel capacity of the legacy cache handler compatibility channel. const cache_handler_channel_cap = 200 -// Setting the maximum data size (500 KB) to be stored at memory mode. If this limit is exceeded, cache handler switch to disk mode. +// Setting the maximum data size (500 KB) to be stored at memory mode. If this limit is exceeded, rendered cache switches to disk mode. const max_size_data_in_memory = 500 // Defines the maximum character length for placeholder keys. const max_placeholders_key_size = 50 // Sets the maximum character length for placeholder values. const max_placeholders_value_size = 3000 -// Internal DTM operations utilize microseconds for time management, primarily aims to minimize the risk of collisions during the creation of temporary cache files -// especially in cases of simultaneous requests that do not yet have a cache. +// Internal DTM operations use microseconds to keep cache generation timestamps monotonic. const convert_seconds = i64(1000000) const converted_cache_delay_expiration_at_min = i64(cache_delay_expiration_at_min) * convert_seconds const converted_cache_delay_expiration_at_max = i64(cache_delay_expiration_at_max) * convert_seconds @@ -38,6 +52,8 @@ const message_signature_info = '[Dynamic Template Manager] Info :' const message_signature_error = '[Dynamic Template Manager] Error :' const message_signature_warn = '[Dynamic Template Manager] Warning :' +@[deprecated: 'use x.templating.dtm2 for new code'] +@[deprecated_after: '2999-01-01'] pub enum CacheStorageMode { memory disk @@ -56,6 +72,8 @@ enum TemplateType { text } +@[deprecated: 'use x.templating.dtm2.Manager for new code'] +@[deprecated_after: '2999-01-01'] @[heap] pub struct DynamicTemplateManager { mut: @@ -67,10 +85,12 @@ mut: template_folder string // cache database template_caches shared []TemplateCache = []TemplateCache{} + // Fast lookup from a full template path hash to the active rendered-cache id. + template_cache_ids_by_path_hash map[u64]int = map[u64]int{} // counter for each individual TemplateCache created/updated id_counter int = 1 ch_cache_handler chan TemplateCache = chan TemplateCache{cap: cache_handler_channel_cap} - // 'id_to_handlered' field is used exclusively by the cache handler to update or delete specific 'TemplateCache' in the cache database. + // 'id_to_handlered' is preserved for legacy callers that route cache update/delete work through the old API. id_to_handlered int close_cache_handler bool // Initialisation params options for these two (Compress_html and active_cache_server) @@ -88,12 +108,15 @@ mut: html_file_info shared map[string]HtmlFileInfo = map[string]HtmlFileInfo{} // Indicates whether the cache file storage directory is located in a temporary OS area cache_folder_is_temporary_storage bool - // Handler for all threads used in the DTM + // Handler for legacy cache-handler compatibility work. threads_handler []thread = []thread{} - // This channel used only for CI. Allows to check during CI tests in case of slowness in the creation/management of the cache to allow enough time for it to be done + // This channel is used by legacy tests/callers to observe cache-handler completion. is_ready chan bool = chan bool{cap: 5} - // If despite the synchronization attempt during the cache handler tests nothing happens, cancel the tests targeting the cached data + // If synchronization times out in legacy cache-handler tests, they set this flag. abort_test bool + // Runtime rendering engine used by the compatibility layer. It caches parsed + // template structures only, never rendered HTML responses. + render_engine &dtm2.Manager = unsafe { nil } } // Represent individual template cache in database memory. @@ -102,6 +125,8 @@ struct TemplateCache { mut: id int name string + // Previous cache id replaced by this cache request, only set for update requests. + old_id int // 'path' field contains the full path, name and file extension of targeted HTML template. path string // Checksum of the cached template, which is constructed as follows: the complete HTML content plus its full file path and the timestamp of the cache generation. @@ -111,18 +136,16 @@ mut: // and the checksum is then calculated based on this string. // Given that the value of this checksum is specific to its parent template, there is no issue of collision, and it does not need to be unique. content_checksum string - // Name of the temporary template created to hold the content until the cache manager processes and handles the request accordingly. - // This file is then deleted once it has been utilized. - tmp_name_file string // The timestamp of the last modification of the file (HTML template) last_template_mod i64 // Timestamp of cache generation. generate_at i64 // Timestamp of cache expiration define by user. cache_delay_expiration i64 - html_data []u8 - cache_request CacheRequest - cache_storage_mode CacheStorageMode + // Rendered HTML kept directly in memory when the cache entry uses memory storage. + html_data string + cache_request CacheRequest + cache_storage_mode CacheStorageMode // This field is special as it determines if a cache is obsolete but still in use, allowing the system to redirect to the updated cache. // This enables the eventual deletion of the obsolete cache without causing issues for requests utilizing the cache. id_redirection int @@ -131,8 +154,8 @@ mut: } // Represents controls stored in the 'nbr_of_remaining_template_request' of the DTM. -// It is used to monitor whether a HTML cache template is in use in 'template_caches' and to manage cache deletion procedures -// as soon as they become feasible and if required of course. +// It is used to monitor whether a rendered cache entry is in use and to manage +// deferred deletion when a cache entry is replaced. @[noinit] struct RemainingTemplateRequest { // The id is similar to the one it represents in template_caches. @@ -140,10 +163,10 @@ struct RemainingTemplateRequest { mut: // This value determines the number of ongoing requests using the cache specified by the ID. nbr_of_remaining_request int - // The cache has become obsolete, and permission to dispose of it is granted to the cache manager + // The cache has become obsolete, and permission to dispose of it is granted to the cache lifecycle. need_to_delete bool - // If the cache manager is unable to destroy an obsolete cache (due to it still being actively used), - // then this boolean allows for the retransmission of the request, enabling action to be taken in the near future. + // If an obsolete cache cannot be destroyed because it is still in use, this + // boolean allows deletion to be retried when the active request finishes. need_to_send_delete_request bool } @@ -154,7 +177,14 @@ struct HtmlFileInfo { file_type TemplateType } +struct PlaceholderChecksumValue { + key string + value string +} + // TemplateCacheParams are used to specify cache expiration delay and provide placeholder data for substitution in templates. +@[deprecated: 'use x.templating.dtm2.RenderParams for new code'] +@[deprecated_after: '2999-01-01'] @[params] pub struct TemplateCacheParams { pub: @@ -163,6 +193,8 @@ pub: } // DynamicTemplateManagerInitialisationParams is used with 'initialize' function. (See below at initialize section) +@[deprecated: 'use x.templating.dtm2.ManagerParams for new code'] +@[deprecated_after: '2999-01-01'] @[params] pub struct DynamicTemplateManagerInitialisationParams { pub: @@ -186,11 +218,13 @@ pub: // - active_cache_server: 'type bool' Activate or not the template cache system. ( default is true ) // - test_cache_dir: 'type string' Used by DTM internal tests to override the cache directory. // - test_template_dir: 'type string' Used by DTM internal tests to override the templates directory. +@[deprecated: 'use x.templating.dtm2.initialize for new code'] +@[deprecated_after: '2999-01-01'] pub fn initialize(dtm_init_params DynamicTemplateManagerInitialisationParams) &DynamicTemplateManager { mut dir_path := dtm_init_params.def_cache_path mut dir_html_path := os.join_path('${os.dir(os.executable())}/templates') mut max_size_memory := 0 - mut active_cache_handler := dtm_init_params.active_cache_server + mut active_rendered_cache := dtm_init_params.active_cache_server mut system_ready := true mut cache_temporary_bool := false @@ -200,7 +234,7 @@ pub fn initialize(dtm_init_params DynamicTemplateManagerInitialisationParams) &D if dtm_init_params.test_template_dir != '' { dir_html_path = dtm_init_params.test_template_dir } - if active_cache_handler { + if active_rendered_cache { // Control if cache folder created by user exist if dir_path != '' && os.exists(dir_path) && os.is_dir(dir_path) { // WARNING: When setting the directory for caching files and for testing purposes, @@ -215,18 +249,18 @@ pub fn initialize(dtm_init_params DynamicTemplateManagerInitialisationParams) &D dir_path = os.join_path(os.temp_dir(), 'vcache_dtm') if !os.exists(dir_path) || !os.is_dir(dir_path) { os.mkdir(dir_path) or { - active_cache_handler = false + active_rendered_cache = false eprintln(err.msg()) } } - if active_cache_handler { + if active_rendered_cache { check_and_clear_cache_files(dir_path) or { - active_cache_handler = false + active_rendered_cache = false eprintln(err.msg()) } } // If it is impossible to use a cache directory, the cache system is deactivated, and the user is warned." - if !active_cache_handler { + if !active_rendered_cache { eprintln('${message_signature_warn} The cache storage directory does not exist or has a problem and it was also not possible to use a folder suitable for temporary storage. Therefore, the cache system will be disabled. It is recommended to address the aforementioned issues to utilize the cache system.') } else { cache_temporary_bool = true @@ -252,21 +286,24 @@ pub fn initialize(dtm_init_params DynamicTemplateManagerInitialisationParams) &D } mut dtmi := &DynamicTemplateManager{ - template_cache_folder: dir_path - template_folder: dir_html_path + template_cache_folder: dir_path.clone() + template_folder: dir_html_path.clone() + template_cache_ids_by_path_hash: map[u64]int{} max_size_data_in_memory: max_size_memory compress_html: dtm_init_params.compress_html - active_cache_server: active_cache_handler + active_cache_server: active_rendered_cache c_time: get_current_unix_micro_timestamp() dtm_init_is_ok: system_ready cache_folder_is_temporary_storage: cache_temporary_bool + render_engine: new_dtm2_render_engine(dir_html_path, + dtm_init_params.compress_html) } if system_ready { - // Disable cache handler if user doesn't required. Else, new thread is used to start the cache system. ( By default is ON ) - if active_cache_handler { + if active_rendered_cache { dtmi.threads_handler << spawn dtmi.cache_handler() - dtmi.threads_handler << spawn dtmi.handle_dtm_clock() } + // Rendered-cache work is processed synchronously. This keeps prod/Boehm behavior + // deterministic while preserving DTM's dynamic rendered-cache feature. println('${message_signature} Dynamic Template Manager activated') } else { eprintln('${message_signature_error} Unable to use the Dynamic Template Manager, please refer to the above errors and correct them.') @@ -294,77 +331,45 @@ fn init_cache_block_middleware(cache_dir string, mut dtm &DynamicTemplateManager // To use this function, HTML templates must be located in the 'templates' directory at the project's root. // However, it allows the use of subfolder paths within the 'templates' directory, // enabling users to structure their templates in a way that best suits their project's organization. +@[deprecated: 'use x.templating.dtm2.Manager.expand for new code'] +@[deprecated_after: '2999-01-01'] pub fn (mut tm DynamicTemplateManager) expand(tmpl_path string, tmpl_var TemplateCacheParams) string { - if tm.dtm_init_is_ok { - file_path, tmpl_name, current_content_checksum, tmpl_type := tm.check_tmpl_and_placeholders_size(tmpl_path, - tmpl_var.placeholders) or { return err.msg() } - converted_cache_delay_expiration := i64(tmpl_var.cache_delay_expiration) * convert_seconds - // If cache exist, return necessary fields else, 'is_cache_exist' return false. - is_cache_exist, id, path, mut last_template_mod, gen_at, cache_del_exp, content_checksum := - tm.return_cache_info_isexistent(file_path) - mut html := '' - // Definition of several variables used to assess the need for cache updates.sss - // This determination is based on modifications within the HTML template itself. - // `last_template_mod` is set to 0 and `test_current_template_mod` to `i64(0)` when a new cache needs to be created. - // `test_current_template_mod` is utilized for updating the cache. - mut test_current_template_mod := i64(0) - if last_template_mod == 0 { - // Get last modification timestamp of HTML template to adding info for the creation of cache. - last_template_mod = os.file_last_mod_unix(file_path) - } else { - // Get last modification timestamp of HTML template to compare with cache info already existent. - test_current_template_mod = os.file_last_mod_unix(file_path) - } - - // From this point, all the previously encountered variables are used to determine the routing in rendering the HTML template and creating/using its cache. - cash_req, unique_time := tm.cache_request_route(is_cache_exist, - converted_cache_delay_expiration, last_template_mod, test_current_template_mod, - cache_del_exp, gen_at, tm.c_time, content_checksum, current_content_checksum) - // Each of these match statements aims to provide HTML rendering, but each one sends a specific signal 'cash_req' of type CacheRequest - // or calls the appropriate function for managing the cache of the provided HTML. - match cash_req { - .new { - // Create a new cache - html = tm.create_template_cache_and_display(cash_req, last_template_mod, - unique_time, file_path, tmpl_name, converted_cache_delay_expiration, - tmpl_var.placeholders, current_content_checksum, tmpl_type) - // println('create cache : ${cash_req}') - } - .update, .exp_update { - // Update an existing cache - tm.id_to_handlered = id - html = tm.create_template_cache_and_display(cash_req, test_current_template_mod, - unique_time, file_path, tmpl_name, converted_cache_delay_expiration, - tmpl_var.placeholders, current_content_checksum, tmpl_type) - // println('update cache : ${cash_req}') - } - else { - // Use the provided cache of html template. - html = tm.get_cache(tmpl_name, path, tmpl_var.placeholders) - // println('get cache : ${cash_req}') - } - } - - return html - } else { - tm.stop_cache_handler() + if !tm.dtm_init_is_ok { + tm.disable_cache() eprintln('${message_signature_error} The initialization phase of DTM has failed. Therefore, you cannot use it. Please address the errors and then restart the dtm server.') return internat_server_error } + return tm.expand_with_dtm2_render_engine(tmpl_path, tmpl_var) } -// stop_cache_handler signals the termination of the cache handler. -// It does so by setting 'close_cache_handler' to true, and sending a signal -// through the channel which will trigger a cascading effect to close the cache -// handler thread as well as the DTM clock thread. +// disable_cache disables future rendered-cache storage for this manager. +@[deprecated: 'legacy DTM compatibility only; use x.templating.dtm2 for new code'] +@[deprecated_after: '2999-01-01'] +pub fn (mut tm DynamicTemplateManager) disable_cache() { + tm.stop_legacy_cache_handler() +} + +// stop_cache_handler is kept for backward compatibility with the former cache-server +// API. Rendered-cache work is now synchronous, so stopping it simply disables future +// rendered-cache storage for this manager. +@[deprecated: 'use disable_cache while maintaining legacy DTM code; use x.templating.dtm2 for new code'] +@[deprecated_after: '2999-01-01'] pub fn (mut tm DynamicTemplateManager) stop_cache_handler() { - if tm.active_cache_server { - tm.active_cache_server = false - tm.close_cache_handler = true - tm.ch_cache_handler <- TemplateCache{ - id: 0 - } + tm.stop_legacy_cache_handler() +} + +fn (mut tm DynamicTemplateManager) stop_legacy_cache_handler() { + if !tm.active_cache_server { + return + } + tm.active_cache_server = false + tm.close_cache_handler = true + tm.ch_cache_handler <- TemplateCache{ + id: 0 + } + if tm.threads_handler.len > 0 { tm.threads_handler.wait() + tm.threads_handler = []thread{} } } @@ -426,8 +431,9 @@ fn (mut tm DynamicTemplateManager) check_tmpl_and_placeholders_size(f_path strin rlock tm.html_file_info { if mapped_html_info := tm.html_file_info[f_path] { - html_file = mapped_html_info.file_full_path - file_name = mapped_html_info.file_name + html_file = mapped_html_info.file_full_path.clone() + file_name = mapped_html_info.file_name.clone() + define_file_type = mapped_html_info.file_type } else { need_to_create_entry = true } @@ -453,9 +459,9 @@ fn (mut tm DynamicTemplateManager) check_tmpl_and_placeholders_size(f_path strin define_file_type = TemplateType.text } lock tm.html_file_info { - tm.html_file_info[f_path] = HtmlFileInfo{ - file_full_path: html_file - file_name: file_name + tm.html_file_info[f_path.clone()] = HtmlFileInfo{ + file_full_path: html_file.clone() + file_name: file_name.clone() file_type: define_file_type } } @@ -469,34 +475,38 @@ fn (mut tm DynamicTemplateManager) check_tmpl_and_placeholders_size(f_path strin // If it has, it creates a checksum of this content for analysis. // The checksum is generated by concatenating all dynamic values and applying a fnv1a hash to the resulting string and generate a checksum. if tmpl_var.len > 0 { - mut combined_str := '' - // Control placeholder key and value sizes + mut placeholder_values := []PlaceholderChecksumValue{cap: tmpl_var.len} for key, value in tmpl_var { if key.len > max_placeholders_key_size { eprintln('${message_signature_error} Length of placeholder key "${key}" exceeds the maximum allowed size for template content in file: ${html_file}. Max allowed size: ${max_placeholders_key_size} characters.') return error(internat_server_error) } + mut casted_value := '' match value { string { - if value.len > max_placeholders_value_size { - eprintln('${message_signature_error} Length of placeholder value for key "${key}" exceeds the maximum allowed size for template content in file: ${html_file}. Max allowed size: ${max_placeholders_value_size} characters.') - return error(internat_server_error) - } - combined_str += value + casted_value = value } else { - casted_value := value.str() - if casted_value.len > max_placeholders_value_size { - eprintln('${message_signature_error} Length of placeholder value for key "${key}" exceeds the maximum allowed size for template content in file: ${html_file}. Max allowed size: ${max_placeholders_value_size} characters.') - return error(internat_server_error) - } - - combined_str += casted_value + casted_value = value.str() } } + + if casted_value.len > max_placeholders_value_size { + eprintln('${message_signature_error} Length of placeholder value for key "${key}" exceeds the maximum allowed size for template content in file: ${html_file}. Max allowed size: ${max_placeholders_value_size} characters.') + return error(internat_server_error) + } + placeholder_values << PlaceholderChecksumValue{ + key: key.clone() + value: casted_value.clone() + } + } + placeholder_values.sort(a.key < b.key) + mut combined_str := strings.new_builder(placeholder_values.len * 16) + for placeholder_value in placeholder_values { + combined_str.write_string(placeholder_value.value) } - res_checksum_content = fnv1a.sum64_string(combined_str).str() + res_checksum_content = fnv1a.sum64_string(combined_str.str()).str() } // If all is ok, return full path of template file and filename without extension @@ -504,14 +514,13 @@ fn (mut tm DynamicTemplateManager) check_tmpl_and_placeholders_size(f_path strin } // create_template_cache_and_display is exclusively invoked from `expand`. -// It generates the template rendering and relaying information -// to the cache manager for either the creation or updating of the template cache. -// It begin to starts by ensuring that the cache delay expiration is correctly set by user. +// It generates the template rendering and prepares rendered-cache information +// for either the creation or updating of the template cache. +// It starts by ensuring that the cache delay expiration is correctly set by user. // It then parses the template file, replacing placeholders with actual dynamics/statics values. // If caching is enabled (indicated by a cache delay expiration different from -1) and the template content is valid, // the function constructs a `TemplateCache` request with all the necessary details. -// This request is then sent to the cache handler channel, signaling either the need for a new cache or an update to an existing one. -// The function returns the rendered immediately, without waiting for the cache to be created or updated. +// This request is processed synchronously, so the rendered cache is ready when the function returns. fn (mut tm DynamicTemplateManager) create_template_cache_and_display(tcs CacheRequest, last_template_mod i64, unique_time i64, file_path string, tmpl_name string, cache_delay_expiration i64, placeholders &map[string]DtmMultiTypeMap, current_content_checksum string, tmpl_type TemplateType) string { @@ -522,31 +531,36 @@ fn (mut tm DynamicTemplateManager) create_template_cache_and_display(tcs CacheRe } // Parses the template and stores the rendered output in the variable. See the function itself for more details. mut html := tm.parse_tmpl_file(file_path, tmpl_name, placeholders, tm.compress_html, tmpl_type) - // If caching is enabled and the template content is valid, this section creates a temporary cache file, which is then used by the cache manager. - // If successfully temporary is created, a cache creation/update notification is sent through its dedicated channel to the cache manager + // If caching is enabled and the template content is valid, this section creates or updates + // the rendered cache synchronously. Keeping the rendered string in the current call avoids + // fragile tmp-file ownership/conversion paths in prod builds. if cache_delay_expiration != -1 && html != internat_server_error && tm.active_cache_server { - op_success, tmp_name := tm.create_temp_cache(html, file_path, unique_time) - if op_success { - tm.ch_cache_handler <- TemplateCache{ - id: tm.id_counter - name: tmpl_name - // 'path' field contains the full path, name and file extension of targeted HTML template. - path: file_path - content_checksum: current_content_checksum - tmp_name_file: tmp_name - // Last modified timestamp of HTML template - last_template_mod: last_template_mod - // Unix current local timestamp of cache generation request converted to UTC - generate_at: unique_time - // Defines the cache expiration delay in seconds. This value is added to 'generate_at' to calculate the expiration time of the cache. - cache_delay_expiration: cache_delay_expiration - // The requested routing to define creation or updating cache. - cache_request: tcs - } - // In the context of a cache update, this function is used to signal that the process has finished using the cache information. The 'nbr_of_remaining_request' counter is therefore updated." - if tcs == .update || tcs == .exp_update { - tm.remaining_template_request(false, tm.id_to_handlered) - } + cache_id := tm.id_counter + tm.id_counter++ + old_cache_id := tm.id_to_handlered + cache_request := TemplateCache{ + id: cache_id + name: tmpl_name.clone() + // 'path' field contains the full path, name and file extension of targeted HTML template. + path: file_path.clone() + content_checksum: current_content_checksum.clone() + // Last modified timestamp of HTML template + last_template_mod: last_template_mod + // Unix current local timestamp of cache generation request converted to UTC + generate_at: unique_time + // Defines the cache expiration delay in seconds. This value is added to 'generate_at' to calculate the expiration time of the cache. + cache_delay_expiration: cache_delay_expiration + html_data: html.clone() + // The requested routing to define creation or updating cache. + cache_request: tcs + old_id: old_cache_id + } + mut process_request := cache_request + tm.process_cache_request(mut process_request) + tm.signal_cache_ready() + // In the context of a cache update, this function is used to signal that the process has finished using the cache information. The 'nbr_of_remaining_request' counter is therefore updated." + if tcs == .update || tcs == .exp_update { + tm.remaining_template_request(false, old_cache_id) } } else { // In the context of a template validity error, the 'nbr_of_remaining_request' counter is consistently updated to avoid anomalies in cache management. @@ -556,123 +570,151 @@ fn (mut tm DynamicTemplateManager) create_template_cache_and_display(tcs CacheRe return html } -// create_temp_cache is responsible for creating a temporary cache file, which is subsequently used exclusively by the cache manager. -// It generates a temporary file name using a checksum based on the timestamp and the file path. -// The content is then written to this file, located in the designated cache folder. -// If the operation is successful, a boolean is returned to allow for the sending of a create or modify request to the cache manager. -fn (tm DynamicTemplateManager) create_temp_cache(html &string, f_path string, ts i64) (bool, string) { - // Extracts the base file name with extension from the given file path. - file_name_with_ext := os.base(f_path) - // Removes the file extension, keeping only the name. - file_name := file_name_with_ext.all_before_last('.') - // Combines the timestamp and file path into a single string - combined_str := ts.str() + f_path - // Generates a md5 hash of the combined string for uniqueness - tmp_checksum := md5.hexhash(combined_str) - // Forms the temporary file name using the file name, checksum, and a .tmp extension - tmp_name := '${file_name}_${tmp_checksum}.tmp' - // Creates the full path for the temporary file in the cache folder - cache_path := os.join_path(tm.template_cache_folder, tmp_name) - // Converts the HTML content into a byte array - html_bytes := html.bytes() - mut f := os.create(cache_path) or { - eprintln('${message_signature_error} Cannot create temporary cache file : ${err.msg()}') - return false, '' +// get_cache is exclusively invoked from `expand', retrieves the rendered HTML from the cache. +fn (mut tm DynamicTemplateManager) get_cache(_name string, path string, _placeholders &map[string]DtmMultiTypeMap) string { + mut cache_id := 0 + rlock tm.template_caches { + for value in tm.template_caches { + // If the cache for the specified HTML template is found, perform the following operations: + if value.path == path { + cache_id = value.id + break + } + } } - f.write(html_bytes) or { - eprintln('${message_signature_error} Cannot write in temporary cache file : ${err.msg()}') - f.close() - return false, '' + if cache_id != 0 { + return tm.get_cache_by_id(cache_id) } - f.close() - return true, tmp_name + return '' } -// get_cache is exclusively invoked from `expand', retrieves the rendered HTML from the cache. -fn (mut tm DynamicTemplateManager) get_cache(name string, path string, placeholders &map[string]DtmMultiTypeMap) string { - mut html := '' - // Lock the cache database for writing. +fn (mut tm DynamicTemplateManager) get_cache_by_id(cache_id int) string { + mut cache_exists := false + mut html_data := '' + mut cache_storage_mode := CacheStorageMode.memory + mut disk_cache_path := '' + mut cache_name := '' rlock tm.template_caches { for value in tm.template_caches { - // If the cache for the specified HTML template is found, perform the following operations: - if value.path == path { - match value.cache_storage_mode { - .memory { - // Retrieve the HTML render from the memory cache and convert it to a string. - html = value.html_data.bytestr() - } - .disk { - r_b_html := os.read_bytes(value.cache_full_path_name) or { - eprintln('${message_signature_error} Get_cache() cannot read template cache file ${value.name} : ${err.msg()} ') - return internat_server_error - } - html = r_b_html.bytestr() - } + if value.id == cache_id { + cache_exists = true + html_data = value.html_data.clone() + cache_storage_mode = value.cache_storage_mode + cache_name = value.name.clone() + disk_cache_path = if value.cache_full_path_name != '' { + value.cache_full_path_name.clone() + } else { + tm.cache_disk_path(value.name, value.checksum) } - - // Function is used to signal that the process has finished using the cache information. The 'nbr_of_remaining_request' counter is therefore updated." - tm.remaining_template_request(false, value.id) - return html + break } } } + defer { + if cache_exists { + tm.remaining_template_request(false, cache_id) + } + } + if html_data.len > 0 { + return html_data + } + match cache_storage_mode { + .memory { + return '' + } + .disk { + cached_html := os.read_file(disk_cache_path) or { + eprintln('${message_signature_error} Get_cache() cannot read template cache file ${cache_name} : ${err.msg()} ') + return internat_server_error + } + return cached_html + } + } - return html + return '' } // return_cache_info_isexistent is exclusively used in 'expand' to determine whether a cache exists for the provided HTML template. // If a cache exists, it returns the necessary information for its transformation. If not, it indicates the need to create a new cache. fn (mut tm DynamicTemplateManager) return_cache_info_isexistent(tmpl_path string) (bool, int, string, i64, i64, i64, string) { - // Lock the cache database for writing. - rlock tm.template_caches { - for value in tm.template_caches { - if value.path == tmpl_path { - // This code section handles cache redirection. - // If a cache redirection ID is found, it indicates that the currently used cache is outdated and there's a newer version available. - // The process then seeks to retrieve information from this more recent cache. - // This is done recursively: if the updated cache itself points to an even newer version, the process continues until the most up-to-date cache is found. - // This recursive mechanism ensures that the latest cache data is always used. - if value.id_redirection != 0 { - mut need_goto := false - mut id_value_recursion := value.id_redirection - unsafe { - re_loop: - inner_loop: for val in tm.template_caches { - if val.id == id_value_recursion { - if val.id_redirection != 0 { - id_value_recursion = val.id_redirection - need_goto = true - break inner_loop - } else { - // function is used to signal that the process has begun using the cache information. - tm.remaining_template_request(true, val.id) - return true, val.id, val.path, val.last_template_mod, val.generate_at, val.cache_delay_expiration, val.content_checksum - } - } - } - if need_goto { - need_goto = false - goto re_loop - } + cache_entries := tm.snapshot_template_caches() + mut cache_found := false + mut cache_id := 0 + mut cache_path := '' + mut last_template_mod := i64(0) + mut generate_at := i64(0) + mut cache_delay_expiration := i64(0) + mut content_checksum := '' + path_hash := fnv1a.sum64_string(tmpl_path) + active_cache_id := tm.template_cache_ids_by_path_hash[path_hash] or { 0 } + for value in cache_entries { + if active_cache_id != 0 { + if value.id != active_cache_id { + continue + } + } else if value.path != tmpl_path { + continue + } + // This code section handles cache redirection. + // If a cache redirection ID is found, it indicates that the currently used cache is outdated and there's a newer version available. + // The process then seeks to retrieve information from this more recent cache. + // This is done recursively: if the updated cache itself points to an even newer version, the process continues until the most up-to-date cache is found. + // This recursive mechanism ensures that the latest cache data is always used. + cache_found = true + cache_id = value.id + cache_path = value.path.clone() + last_template_mod = value.last_template_mod + generate_at = value.generate_at + cache_delay_expiration = value.cache_delay_expiration + content_checksum = value.content_checksum.clone() + if value.id_redirection != 0 { + mut id_value_recursion := value.id_redirection + for { + mut found_redirect := false + for val in cache_entries { + if val.id == id_value_recursion { + cache_id = val.id + cache_path = val.path.clone() + last_template_mod = val.last_template_mod + generate_at = val.generate_at + cache_delay_expiration = val.cache_delay_expiration + content_checksum = val.content_checksum.clone() + found_redirect = true + break } - // No cache redirection, get cache current information. - } else { - // function is used to signal that the process has begun using the cache information. - tm.remaining_template_request(true, value.id) - return true, value.id, value.path, value.last_template_mod, value.generate_at, value.cache_delay_expiration, value.content_checksum } + if !found_redirect { + break + } + mut next_redirection := 0 + for val in cache_entries { + if val.id == cache_id { + next_redirection = val.id_redirection + break + } + } + if next_redirection == 0 { + break + } + id_value_recursion = next_redirection } } + break + } + if cache_found { + // Function is used to signal that the process has begun using the cache information. + tm.remaining_template_request(true, cache_id) + return true, cache_id, cache_path, last_template_mod, generate_at, cache_delay_expiration, content_checksum } // No existing cache, need to create it. return false, 0, '', 0, 0, 0, '' } -// remaining_template_request manages the counter in 'nbr_of_remaining_template_request', which tracks the number of requests that have started or finished for a specific cache. -// Moreover, this function sends a cache deletion callback request when the cache manager had previously been instructed to delete the cache but was unable to do because, -// it was still in use. +// remaining_template_request manages the counter in 'nbr_of_remaining_template_request', +// which tracks the number of requests that have started or finished for a specific +// rendered-cache entry. If a replaced cache is no longer in use, it is deleted. fn (mut tm DynamicTemplateManager) remaining_template_request(b bool, v int) { - // Lock the remaining template request process for reading and writing. + mut delete_request_id := 0 lock tm.nbr_of_remaining_template_request { for key, r_request in tm.nbr_of_remaining_template_request { if r_request.id == v { @@ -683,13 +725,9 @@ fn (mut tm DynamicTemplateManager) remaining_template_request(b bool, v int) { // if false, Decrements the count of active cache requests. tm.nbr_of_remaining_template_request[key].nbr_of_remaining_request -= 1 // Checks if the number of active requests is zero or less and if there's a pending delete request for the cache. - // If yes, request is sent to the cache handler on this own channel. - if r_request.nbr_of_remaining_request <= 0 - && r_request.need_to_send_delete_request { - tm.ch_cache_handler <- TemplateCache{ - id: r_request.id - cache_request: .delete - } + if tm.nbr_of_remaining_template_request[key].nbr_of_remaining_request <= 0 + && tm.nbr_of_remaining_template_request[key].need_to_send_delete_request { + delete_request_id = r_request.id } } @@ -697,128 +735,156 @@ fn (mut tm DynamicTemplateManager) remaining_template_request(b bool, v int) { } } } + if delete_request_id != 0 { + mut delete_request := TemplateCache{ + id: delete_request_id + cache_request: .delete + } + tm.process_cache_request(mut delete_request) + } +} + +// process_cache_request stores or deletes one rendered-cache entry synchronously. +// Small rendered outputs are kept in memory; larger outputs are written to disk. +fn (mut tm DynamicTemplateManager) process_cache_request(mut tc TemplateCache) bool { + tc.ensure_owned_data() + + // Determine if the request is a duplicate. If so, the cache creation/update request is ignored. + is_duplicate_request := tm.chandler_prevent_cache_duplicate_request(tc) + if is_duplicate_request { + return true + } + + if tc.cache_request != .delete { + rendered_html := tc.html_data.clone() + if tc.html_data.len <= (tm.max_size_data_in_memory * 1024) { + tc.cache_storage_mode = .memory + } else { + tc.cache_storage_mode = .disk + } + + combined_str := rendered_html + tc.path + tc.generate_at.str() + tc.checksum = md5.hexhash(combined_str).clone() + + match tc.cache_storage_mode { + .memory { + tc.html_data = rendered_html.clone() + } + .disk { + cache_file_path := tm.cache_disk_path(tc.name, tc.checksum) + os.write_file(cache_file_path, rendered_html) or { + eprintln('${message_signature_error} Rendered cache: failed to write cache file: ${err.msg()}') + return false + } + tc.cache_full_path_name = cache_file_path.clone() + // Keep a hot in-memory copy even for disk-backed entries. The disk file + // preserves compatibility, while cache hits avoid fragile string metadata + // roundtrips in optimized prod builds. + tc.html_data = rendered_html.clone() + } + } + } + + if tc.cache_request != .delete { + stable_cache_entry := tc.clone_cache_entry() + lock tm.template_caches { + tm.template_caches << stable_cache_entry + } + path_hash := fnv1a.sum64_string(stable_cache_entry.path) + tm.template_cache_ids_by_path_hash[path_hash] = stable_cache_entry.id + } + if tc.cache_request == .new { + tm.chandler_remaining_cache_template_used(tc.cache_request, tc.id, tc.old_id) + } else { + test_b := tm.chandler_remaining_cache_template_used(tc.cache_request, tc.id, tc.old_id) + if test_b { + clear_cache_id := if tc.cache_request == .delete { tc.id } else { tc.old_id } + key, is_success := tm.chandler_clear_specific_cache(clear_cache_id) + if !is_success { + return false + } + lock tm.template_caches { + tm.template_caches.delete(key) + } + } + } + return true +} + +fn (tm &DynamicTemplateManager) cache_disk_path(name string, checksum string) string { + cache_file_name := '${name}_${checksum}.cache' + return os.join_path(tm.template_cache_folder, cache_file_name).clone() +} + +fn (mut tc TemplateCache) ensure_owned_strings() { + tc.name = tc.name.clone() + tc.path = tc.path.clone() + tc.checksum = tc.checksum.clone() + tc.content_checksum = tc.content_checksum.clone() + tc.cache_full_path_name = tc.cache_full_path_name.clone() +} + +fn (mut tc TemplateCache) ensure_owned_data() { + tc.ensure_owned_strings() + tc.html_data = tc.html_data.clone() +} + +fn (tc TemplateCache) clone_cache_entry() TemplateCache { + return TemplateCache{ + id: tc.id + name: tc.name.clone() + old_id: tc.old_id + path: tc.path.clone() + checksum: tc.checksum.clone() + content_checksum: tc.content_checksum.clone() + last_template_mod: tc.last_template_mod + generate_at: tc.generate_at + cache_delay_expiration: tc.cache_delay_expiration + html_data: tc.html_data.clone() + cache_request: tc.cache_request + cache_storage_mode: tc.cache_storage_mode + id_redirection: tc.id_redirection + cache_full_path_name: tc.cache_full_path_name.clone() + } +} + +fn (tm &DynamicTemplateManager) snapshot_template_caches() []TemplateCache { + rlock tm.template_caches { + mut entries := []TemplateCache{cap: tm.template_caches.len} + for entry in tm.template_caches { + entries << entry.clone_cache_entry() + } + return entries + } +} + +fn (mut tm DynamicTemplateManager) signal_cache_ready() { + $if test { + select { + tm.is_ready <- true {} + else {} + } + } } -// fn (mut DynamicTemplateManager) cache_handler() -// -// This function serves as the core handler for managing the cache within the DTM. -// It continuously listens for cache update requests and processes them accordingly. -// When a cache request is received, it either creates or updates or deletes the cache based on the request type. -// The function ensures that duplicate cache requests are avoided and handles the closing of the cache handler. -// It is necessary to restart the entire application in case the manager closes. -// The manager handles cache operations in a multithreaded context, accepting up to a list of 200 operation requests. (Define in 'cache_handler_channel_cap' constant) -// The HTML rendering is stored as a u8 array. -// -// TODO: Currently, the cache manager stops when it encounters an internal error requiring a restart of the program. -// ( it is designed to ignore external errors since these are already handled in a way that ensures no cache processing requests are affected ), -// A recovery system will need to be implemented to ensure service continuity. -// fn (mut tm DynamicTemplateManager) cache_handler() { defer { - // If cause is an internal cache handler error - tm.active_cache_server = false - // Close channel if handler is stopped tm.ch_cache_handler.close() tm.is_ready.close() - tm.ch_stop_dtm_clock <- true } for { select { - // Continuously listens until a request is received through the dedicated channel. - mut tc := <-tm.ch_cache_handler { - // Close handler if asked. + tc := <-tm.ch_cache_handler { if tm.close_cache_handler { eprintln('${message_signature_info} Cache manager has been successfully stopped. Please consider restarting the application if needed.') - break + return } - f_path_tmp := os.join_path(tm.template_cache_folder, tc.tmp_name_file) - - // determine if the requests passed to the manager are not duplicate requests. If so, the temporary file will be destroyed as part of a cache creation/update request. - if !tm.chandler_prevent_cache_duplicate_request(tc) { - if tc.cache_request != .delete { - // Determines the size of the template content to decide where to store it (in memory or on disk). - // It retrieves the content from the temporary file and then forms a unique checksum for the final name of the cache file. - // The file size is compared to the maximum allowable data size in memory. - // If the size is within the limit, the cache is stored in memory; otherwise, it's stored on disk. - // The unique checksum is generated by hashing the file data, its path, and the cache generation timestamp using md5. - tmp_file_size := os.file_size(f_path_tmp) - if tmp_file_size <= (tm.max_size_data_in_memory * 1024) { - tc.cache_storage_mode = .memory - } else { - tc.cache_storage_mode = .disk - } - file_data := os.read_bytes(f_path_tmp) or { - eprintln('${message_signature_error} Cache Handler : Failed to read tmp file, cache server will be stopped, you need to fix and restart application: ${err.msg()}') - break - } - - combined_str := file_data.str() + tc.path + tc.generate_at.str() - tc.checksum = md5.hexhash(combined_str) - - match tc.cache_storage_mode { - .memory { - // If the cache is stored in memory, the temporary file is destroyed. - tc.html_data = file_data - os.rm(f_path_tmp) or { - eprintln('${message_signature_error} Cache Handler : While deleting the tmp cache file: "${f_path_tmp}", cache server will be stopped, you need to fix and restart application: ${err.msg()}') - break - } - } - .disk { - // If the cache is stored on disk, the temporary file is renamed to become the definitive cache of the current version of the HTML template. - tc.cache_full_path_name = os.join_path(tm.template_cache_folder, - '${tc.name}_${tc.checksum}.cache') - os.mv(f_path_tmp, tc.cache_full_path_name) or { - eprintln('${message_signature_error} Cache Handler : Failed to rename tmp file, cache server will be stopped, you need to fix and restart application: ${err.msg()}') - break - } - } - } - } - // Lock the cache database for reading and writing. - lock tm.template_caches { - if tc.cache_request != .delete { - // Include Cache information in database. - tm.template_caches << tc - - $if test { - tm.is_ready <- true - } - } - if tc.cache_request == .new { - tm.chandler_remaining_cache_template_used(tc.cache_request, tc.id, - tm.id_to_handlered) - // Increment ID counter for the next creation/update cache request - tm.id_counter++ - } else { - if tc.cache_request != .delete { - tm.id_counter++ - } - // This function allows the cache manager to handle what happens in 'nbr_of_remaining_template_request' and - // act accordingly for the creation, update, or destruction of the cache. - test_b := tm.chandler_remaining_cache_template_used(tc.cache_request, - tc.id, tm.id_to_handlered) - if test_b { - // Finding position of cache in database ( If disk mode, cache is erased ) - key, is_success := - tm.chandler_clear_specific_cache(tm.id_to_handlered) - if !is_success { - break - } - // Delete in database. - tm.template_caches.delete(key) - - $if test { - tm.is_ready <- true - } - } - } - } - } else if tc.cache_request != .delete { - os.rm(f_path_tmp) or { - eprintln('${message_signature_warn} Cache Handler : Cannot deleting the unused tmp cache file: "${f_path_tmp}" : ${err.msg()}') - } + mut request := tc + if request.cache_request != .delete && request.old_id == 0 { + request.old_id = tm.id_to_handlered } + tm.process_cache_request(mut request) + tm.signal_cache_ready() } } } @@ -826,11 +892,11 @@ fn (mut tm DynamicTemplateManager) cache_handler() { // fn (DynamicTemplateManager) chandler_prevent_cache_duplicate_request(&TemplateCache) return bool // -// Exclusively used by the cache handler, assesses whether a cache request is a duplicate, +// Used by rendered-cache processing to assess whether a cache request is a duplicate, // based on the type of cache request (.new, .update, .exp_update, .delete) and the existing data. // Returns true to indicate a duplicate request, which will be ignored, and false otherwise. // -fn (tm DynamicTemplateManager) chandler_prevent_cache_duplicate_request(tc &TemplateCache) bool { +fn (tm &DynamicTemplateManager) chandler_prevent_cache_duplicate_request(tc &TemplateCache) bool { match tc.cache_request { .new { for value in tm.template_caches { @@ -878,19 +944,24 @@ fn (tm DynamicTemplateManager) chandler_prevent_cache_duplicate_request(tc &Temp // fn (mut DynamicTemplateManager) chandler_clear_specific_cache(int) return (int, bool) // -// Exclusively associated with the cache handler, is used to remove specific cache information from the database when necessary. +// Used to remove specific rendered-cache information from the database when necessary. // It identifies the target 'TemplateCache' by its id in the array database and deletes its corresponding cache file in 'disk mode'. // fn (mut tm DynamicTemplateManager) chandler_clear_specific_cache(id int) (int, bool) { for key, value in tm.template_caches { if value.id == id { + path_hash := fnv1a.sum64_string(value.path) + if active_cache_id := tm.template_cache_ids_by_path_hash[path_hash] { + if active_cache_id == id { + tm.template_cache_ids_by_path_hash.delete(path_hash) + } + } match value.cache_storage_mode { .memory {} .disk { - file_path := os.join_path(tm.template_cache_folder, - '${value.name}_${value.checksum}.cache') + file_path := tm.cache_disk_path(value.name, value.checksum) os.rm(file_path) or { - eprintln('${message_signature_error} While deleting the specific cache file: ${file_path}, cache server will be stopped, you need to fix and restart application: : ${err.msg()}') + eprintln('${message_signature_error} While deleting the specific cache file: ${file_path}: ${err.msg()}') break } } @@ -904,7 +975,7 @@ fn (mut tm DynamicTemplateManager) chandler_clear_specific_cache(id int) (int, b // fn (mut DynamicTemplateManager) chandler_remaining_cache_template_used(CacheRequest, int, int) return bool // -// Exclusively associated with the cache handler for managing the lifecycle of cache requests in 'nbr_of_remaining_template_request'. +// Manages the lifecycle of rendered-cache requests in 'nbr_of_remaining_template_request'. // For each cache request (creation, update, or deletion), it updates the status of ongoing requests and decides on necessary actions. // The function returns a boolean value: if true, it authorizes the destruction of the expired cache, ensuring that it's only removed when no longer in use. // If false, it indicates that the cache cannot yet be destroyed due to ongoing usage. @@ -921,19 +992,23 @@ fn (mut tm DynamicTemplateManager) chandler_remaining_cache_template_used(cr Cac .update, .exp_update { // Marks the old cache as obsolete and adds a request for the new cache. // If the old cache is no longer in use, it is immediately deleted. Otherwise, a flag is set for deferred deletion of the cache as soon as feasible - for key, mut value in tm.nbr_of_remaining_template_request { - if value.id == old_id { - value.need_to_delete = true - tm.nbr_of_remaining_template_request << RemainingTemplateRequest{ - id: id - } + for key := 0; key < tm.nbr_of_remaining_template_request.len; key++ { + if tm.nbr_of_remaining_template_request[key].id == old_id { + tm.nbr_of_remaining_template_request[key].need_to_delete = true // If possible, immediately deleted request - if value.nbr_of_remaining_request <= 0 && value.need_to_delete == true { + if tm.nbr_of_remaining_template_request[key].nbr_of_remaining_request <= 0 + && tm.nbr_of_remaining_template_request[key].need_to_delete == true { tm.nbr_of_remaining_template_request.delete(key) + tm.nbr_of_remaining_template_request << RemainingTemplateRequest{ + id: id + } return true // else, set for deferred deletion of the cache as soon as feasible } else { - value.need_to_send_delete_request = true + tm.nbr_of_remaining_template_request[key].need_to_send_delete_request = true + tm.nbr_of_remaining_template_request << RemainingTemplateRequest{ + id: id + } } break } @@ -942,12 +1017,12 @@ fn (mut tm DynamicTemplateManager) chandler_remaining_cache_template_used(cr Cac } .delete { // Removes the cache request from the list if it's no longer in use. - for key, value in tm.nbr_of_remaining_template_request { - if value.id == id { - if value.nbr_of_remaining_request <= 0 && value.need_to_delete == true { + for key := 0; key < tm.nbr_of_remaining_template_request.len; key++ { + if tm.nbr_of_remaining_template_request[key].id == id { + if tm.nbr_of_remaining_template_request[key].nbr_of_remaining_request <= 0 + && tm.nbr_of_remaining_template_request[key].need_to_delete == true { tm.nbr_of_remaining_template_request.delete(key) } - break } } @@ -955,6 +1030,7 @@ fn (mut tm DynamicTemplateManager) chandler_remaining_cache_template_used(cr Cac else {} } } + return true } @@ -983,21 +1059,36 @@ fn (mut tm DynamicTemplateManager) parse_tmpl_file(file_path string, tmpl_name s // Performs a light compression of the HTML output by removing usless spaces, newlines, and tabs if user selected this option. if is_compressed && tmpl_type == TemplateType.html && tmpl_ != internat_server_error { - tmpl_ = tmpl_.replace_each(['\n', '', '\t', '', ' ', ' ']) - mut r := regex.regex_opt(r'>(\s+)<') or { - tm.stop_cache_handler() - eprintln('${message_signature_error} with regular expression for HTML light compression in parse_tmpl_file() function. Please check the syntax of the regex pattern : ${err.msg()}') - return internat_server_error - } - tmpl_ = r.replace(tmpl_, '><') - for tmpl_.contains(' ') { - tmpl_ = tmpl_.replace(' ', ' ') - } + tmpl_ = compress_html_output(tmpl_) } return tmpl_ } +fn compress_html_output(html string) string { + mut compressed := html.replace('\n', '').replace('\t', '') + for compressed.contains(' ') { + compressed = compressed.replace(' ', ' ') + } + mut result := strings.new_builder(compressed.len) + mut i := 0 + for i < compressed.len { + result.write_u8(compressed[i]) + if compressed[i] == `>` { + mut j := i + 1 + for j < compressed.len && compressed[j] == ` ` { + j++ + } + if j < compressed.len && compressed[j] == `<` { + i = j + continue + } + } + i++ + } + return result.str() +} + // fn check_if_cache_delay_iscorrect(i64, string) return ! // // Validates the user-specified cache expiration delay for templates. @@ -1020,23 +1111,21 @@ fn check_if_cache_delay_iscorrect(cde i64, tmpl_name string) ! { // to decide whether to create a new cache, update an existing or delivered a valid cache content. // fn (mut tm DynamicTemplateManager) cache_request_route(is_cache_exist bool, neg_cache_delay_expiration i64, - last_template_mod i64, test_current_template_mod i64, cache_del_exp i64, gen_at i64, c_time i64, content_checksum string, - current_content_checksum string) (CacheRequest, i64) { + last_template_mod i64, test_current_template_mod i64, cache_del_exp i64, gen_at i64, c_time i64, + content_checksum string, current_content_checksum string) (CacheRequest, i64) { + current_ts := if c_time > 0 { c_time } else { tm.next_cache_timestamp() } if !is_cache_exist || neg_cache_delay_expiration == -1 { // Require cache creation - unique_ts := get_current_unix_micro_timestamp() - tm.c_time = unique_ts + unique_ts := tm.next_cache_timestamp() return CacheRequest.new, unique_ts } else if last_template_mod < test_current_template_mod || content_checksum != current_content_checksum { // Requires cache update as the template has been modified since the last time. it can be the template itself or its dynamic content. - unique_ts := get_current_unix_micro_timestamp() - tm.c_time = unique_ts + unique_ts := tm.next_cache_timestamp() return CacheRequest.update, unique_ts - } else if cache_del_exp != 0 && (gen_at + cache_del_exp) < tm.c_time { - unique_ts := get_current_unix_micro_timestamp() - tm.c_time = unique_ts + } else if cache_del_exp != 0 && (gen_at + cache_del_exp) < current_ts { // Requires cache update as the cache expiration delay has elapsed. + unique_ts := tm.next_cache_timestamp() return CacheRequest.exp_update, unique_ts } else { // Returns valid cached content, no update or creation necessary. @@ -1044,59 +1133,14 @@ fn (mut tm DynamicTemplateManager) cache_request_route(is_cache_exist bool, neg_ } } -// fn (mut tm DynamicTemplateManager) handle_dtm_clock() -// -// Manages the internal clock. It periodically updates ( 4 minutes minimum, if there has been a recent update elsewhere in the system, it will be taken into account by the clock handler. ) -// The goal is ensuring that the DTM's internal time is maintained, especially during prolonged periods of inactivity on the website, -// this can potentially lead to cache expiration issues if there is zero traffic for a while." -// - -// Minimum update interval ( in seconds ) set to 4 minutes minimum_wait_time_until_next_update -const update_duration = 240 - -fn (mut tm DynamicTemplateManager) handle_dtm_clock() { - mut need_to_close := false - defer { - tm.ch_stop_dtm_clock.close() - eprintln('${message_signature_info} DTM clock handler has been successfully stopped.') - } - - for { - // Calculate the remaining time until the next update. - current_time := get_current_unix_micro_timestamp() / convert_seconds - mut time_since_last_update := int(current_time - (tm.c_time / convert_seconds)) - mut minimum_wait_time_until_next_update := update_duration - - // Update DTM clock if update interval exceeded otherwise, set next check based on time since last update - if time_since_last_update >= update_duration { - tm.c_time = current_time * convert_seconds - } else { - if time_since_last_update < 0 { - time_since_last_update = 0 - } - minimum_wait_time_until_next_update = (update_duration - time_since_last_update) + - update_duration - } - - // Wait until the next update interval or until a stop signal is received. - for elapsed_time := 0; elapsed_time < minimum_wait_time_until_next_update; elapsed_time++ { - select { - _ := <-tm.ch_stop_dtm_clock { - need_to_close = true - break - } - else { - // Attendre une seconde - time.sleep(1 * time.second) - } - } - } - if need_to_close { - break - } - // Reset wait time for next cycle. - minimum_wait_time_until_next_update = update_duration +fn (mut tm DynamicTemplateManager) next_cache_timestamp() i64 { + current_ts := get_current_unix_micro_timestamp() + if current_ts <= tm.c_time { + tm.c_time++ + } else { + tm.c_time = current_ts } + return tm.c_time } // fn get_current_unix_timestamp() return i64 diff --git a/vlib/x/templating/dtm/dynamic_template_manager_behavior_test.v b/vlib/x/templating/dtm/dynamic_template_manager_behavior_test.v new file mode 100644 index 00000000000000..e776f81a6395e9 --- /dev/null +++ b/vlib/x/templating/dtm/dynamic_template_manager_behavior_test.v @@ -0,0 +1,333 @@ +// vtest retry: 3 +module dtm + +import os +import time + +const behavior_temp_dtm_dir = 'dynamic_template_manager_behavior_test' +const behavior_temp_cache_dir = 'vcache_dtm' +const behavior_temp_templates_dir = 'templates' +const behavior_vtmp_dir = os.vtmp_dir() + +fn behavior_test_root_dir() string { + return os.join_path(behavior_vtmp_dir, '${behavior_temp_dtm_dir}_${os.getpid()}') +} + +fn testsuite_begin() { + temp_folder := behavior_test_root_dir() + os.rmdir_all(temp_folder) or {} + os.mkdir_all(os.join_path(temp_folder, behavior_temp_cache_dir))! + os.mkdir_all(os.join_path(temp_folder, behavior_temp_templates_dir))! + + templates_path := os.join_path(temp_folder, behavior_temp_templates_dir) + os.write_file(os.join_path(templates_path, 'page.html'), [ + '', + '@title', + '

@body

', + '

@count

', + ].join('\n'))! + os.write_file(os.join_path(templates_path, 'page.txt'), [ + 'Title: @title', + 'Body: @body', + ].join('\n'))! + os.write_file(os.join_path(templates_path, 'layout.html'), [ + '
Start
', + "@include 'partial'", + '', + ].join('\n'))! + os.write_file(os.join_path(templates_path, 'partial.html'), '
@title
')! + os.write_file(os.join_path(templates_path, 'cache.html'), '
@title
')! + os.write_file(os.join_path(templates_path, 'cache_file_update.html'), '
@title
')! + os.write_file(os.join_path(templates_path, 'no_store.html'), '
@title
')! + os.write_file(os.join_path(templates_path, 'forever.html'), '
@title
')! + os.write_file(os.join_path(templates_path, 'disk_cache.html'), '
@title
')! + os.write_file(os.join_path(templates_path, 'include_html.html'), '
@content
')! + os.write_file(os.join_path(templates_path, 'include_text.txt'), 'Text: @content')! + os.write_file(os.join_path(templates_path, 'invalid.css'), 'body { color: red; }')! +} + +fn testsuite_end() { + os.rmdir_all(behavior_test_root_dir()) or {} +} + +fn new_behavior_dtm(active_cache_server bool, compress_html bool, max_size_data_in_mem int) &DynamicTemplateManager { + temp_folder := behavior_test_root_dir() + return initialize( + active_cache_server: active_cache_server + compress_html: compress_html + max_size_data_in_mem: max_size_data_in_mem + test_cache_dir: os.join_path(temp_folder, behavior_temp_cache_dir) + test_template_dir: os.join_path(temp_folder, behavior_temp_templates_dir) + ) +} + +fn behavior_template_path(file_name string) string { + return os.join_path(behavior_test_root_dir(), behavior_temp_templates_dir, file_name) +} + +fn rendered_cache_count(dtmi &DynamicTemplateManager) int { + rlock dtmi.template_caches { + return dtmi.template_caches.len + } +} + +fn test_expand_escapes_html_placeholders_by_default() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := { + 'title': DtmMultiTypeMap('Hello ') + 'body': DtmMultiTypeMap('') + 'count': DtmMultiTypeMap(7) + } + + rendered := dtmi.expand(behavior_template_path('page.html'), placeholders: &placeholders) + + assert rendered.contains('Hello <V>') + assert rendered.contains('

<script>alert(1)</script>

') + assert rendered.contains('

7

') +} + +fn test_expand_does_not_double_escape_placeholder_values_containing_dollar() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := { + 'title': DtmMultiTypeMap('Cost $5 ') + 'body': DtmMultiTypeMap('Body') + 'count': DtmMultiTypeMap(1) + } + + rendered := dtmi.expand(behavior_template_path('page.html'), placeholders: &placeholders) + + assert rendered.contains('Cost $5 <V>') +} + +fn test_expand_escapes_html_entities_and_quotes() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := { + 'title': DtmMultiTypeMap('A & B "quoted" \'single\'') + 'body': DtmMultiTypeMap('Body') + 'count': DtmMultiTypeMap(1) + } + + rendered := dtmi.expand(behavior_template_path('page.html'), placeholders: &placeholders) + + assert rendered.contains('A & B "quoted" 'single'') +} + +fn test_expand_escapes_text_templates() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := { + 'title': DtmMultiTypeMap('Plain ') + 'body': DtmMultiTypeMap('<strong>raw text</strong>') + } + + rendered := dtmi.expand(behavior_template_path('page.txt'), placeholders: &placeholders) + + assert rendered.contains('Title: Plain <Title>') + assert rendered.contains('Body: <strong>raw text</strong>') +} + +fn test_expand_compresses_html_without_regex_failure() { + mut dtmi := new_behavior_dtm(false, true, max_size_data_in_memory) + placeholders := { + 'title': DtmMultiTypeMap('Compressed') + 'body': DtmMultiTypeMap('Body') + 'count': DtmMultiTypeMap(1) + } + + rendered := dtmi.expand(behavior_template_path('page.html'), placeholders: &placeholders) + + assert rendered != internat_server_error + assert !rendered.contains('\n') + assert rendered.contains('<!doctype html><title>Compressed

Body

1

') +} + +fn test_expand_with_empty_placeholders_keeps_template_renderable() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := map[string]DtmMultiTypeMap{} + + rendered := dtmi.expand(behavior_template_path('page.html'), placeholders: &placeholders) + + assert rendered.contains('$title') + assert rendered.contains('

$body

') + assert rendered.contains('

$count

') +} + +fn test_expand_resolves_relative_includes() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := { + 'title': DtmMultiTypeMap('Included ') + } + + rendered := dtmi.expand(behavior_template_path('layout.html'), placeholders: &placeholders) + + assert rendered.contains('<header>Start</header>') + assert rendered.contains('<section>Included <Title></section>') + assert rendered.contains('<footer>End</footer>') +} + +fn test_includehtml_preserves_allowed_tags_and_escapes_unlisted_tags() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := { + 'content_#includehtml': DtmMultiTypeMap('<span>allowed</span><script>blocked()</script>') + } + + rendered := + dtmi.expand(behavior_template_path('include_html.html'), placeholders: &placeholders) + + assert rendered.contains('<article><span>allowed</span><script>blocked()</script></article>') +} + +fn test_includehtml_is_still_escaped_in_text_templates() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := { + 'content_#includehtml': DtmMultiTypeMap('<span>text</span>') + } + + rendered := dtmi.expand(behavior_template_path('include_text.txt'), placeholders: &placeholders) + + assert rendered.contains('Text: <span>text</span>') +} + +fn test_expand_returns_internal_error_for_missing_template() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := map[string]DtmMultiTypeMap{} + + rendered := dtmi.expand(behavior_template_path('missing.html'), placeholders: &placeholders) + + assert rendered == internat_server_error +} + +fn test_expand_returns_internal_error_for_invalid_template_extension() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + placeholders := map[string]DtmMultiTypeMap{} + + rendered := dtmi.expand(behavior_template_path('invalid.css'), placeholders: &placeholders) + + assert rendered == internat_server_error +} + +fn test_expand_without_cache_renders_fresh_placeholder_content() { + mut dtmi := new_behavior_dtm(false, false, max_size_data_in_memory) + first_placeholders := { + 'title': DtmMultiTypeMap('first') + } + second_placeholders := { + 'title': DtmMultiTypeMap('second') + } + + first := dtmi.expand(behavior_template_path('cache.html'), placeholders: &first_placeholders) + second := dtmi.expand(behavior_template_path('cache.html'), placeholders: &second_placeholders) + + assert first.contains('<main>first</main>') + assert second.contains('<main>second</main>') +} + +fn test_expand_with_cache_updates_when_placeholder_content_changes() { + mut dtmi := new_behavior_dtm(true, false, max_size_data_in_memory) + defer { + dtmi.disable_cache() + } + first_placeholders := { + 'title': DtmMultiTypeMap('cached first') + } + second_placeholders := { + 'title': DtmMultiTypeMap('cached second') + } + + first := dtmi.expand(behavior_template_path('cache.html'), placeholders: &first_placeholders) + second := dtmi.expand(behavior_template_path('cache.html'), placeholders: &first_placeholders) + third := dtmi.expand(behavior_template_path('cache.html'), placeholders: &second_placeholders) + + assert first.contains('<main>cached first</main>') + assert second == first + assert third.contains('<main>cached second</main>') +} + +fn test_expand_with_legacy_cache_enabled_uses_new_parsed_template_engine() { + mut dtmi := new_behavior_dtm(true, false, max_size_data_in_memory) + defer { + dtmi.disable_cache() + } + placeholders := { + 'title': DtmMultiTypeMap('stable cached value') + } + + first := dtmi.expand(behavior_template_path('cache.html'), placeholders: &placeholders) + second := dtmi.expand(behavior_template_path('cache.html'), placeholders: &placeholders) + + assert first == second + assert rendered_cache_count(dtmi) == 0 + assert dtmi.render_engine.compiled_template_count() == 1 +} + +fn test_expand_with_cache_updates_when_template_file_changes() { + mut dtmi := new_behavior_dtm(true, false, max_size_data_in_memory) + defer { + dtmi.disable_cache() + } + template_path := behavior_template_path('cache_file_update.html') + placeholders := { + 'title': DtmMultiTypeMap('file driven') + } + + first := dtmi.expand(template_path, placeholders: &placeholders) + time.sleep(1100 * time.millisecond) + os.write_file(template_path, '<section>@title</section>')! + second := dtmi.expand(template_path, placeholders: &placeholders) + + assert first.contains('<main>file driven</main>') + assert second.contains('<section>file driven</section>') +} + +fn test_cache_delay_minus_one_skips_rendered_cache_storage() { + mut dtmi := new_behavior_dtm(true, false, max_size_data_in_memory) + defer { + dtmi.disable_cache() + } + placeholders := { + 'title': DtmMultiTypeMap('no store') + } + + rendered := dtmi.expand(behavior_template_path('no_store.html'), + placeholders: &placeholders + cache_delay_expiration: -1 + ) + + assert rendered.contains('<main>no store</main>') + assert rendered_cache_count(dtmi) == 0 +} + +fn test_cache_delay_zero_is_accepted_without_rendered_cache_storage() { + mut dtmi := new_behavior_dtm(true, false, max_size_data_in_memory) + defer { + dtmi.disable_cache() + } + placeholders := { + 'title': DtmMultiTypeMap('forever') + } + + rendered := dtmi.expand(behavior_template_path('forever.html'), + placeholders: &placeholders + cache_delay_expiration: 0 + ) + + assert rendered.contains('<main>forever</main>') + assert rendered_cache_count(dtmi) == 0 +} + +fn test_memory_limit_zero_does_not_create_rendered_disk_cache() { + mut dtmi := new_behavior_dtm(true, false, 0) + defer { + dtmi.disable_cache() + } + placeholders := { + 'title': DtmMultiTypeMap('disk mode') + } + + rendered := dtmi.expand(behavior_template_path('disk_cache.html'), placeholders: &placeholders) + + assert rendered.contains('<main>disk mode</main>') + assert rendered_cache_count(dtmi) == 0 + for file_name in os.ls(dtmi.template_cache_folder)! { + assert !file_name.ends_with('.cache') + } +} diff --git a/vlib/x/templating/dtm/dynamic_template_manager_cache_system_test.v b/vlib/x/templating/dtm/dynamic_template_manager_cache_system_test.v index 031598c45ed053..aec84fa27c5690 100644 --- a/vlib/x/templating/dtm/dynamic_template_manager_cache_system_test.v +++ b/vlib/x/templating/dtm/dynamic_template_manager_cache_system_test.v @@ -62,12 +62,10 @@ fn test_get_cache() { defer { dtmi.stop_cache_handler() } - if !dtmi.abort_test { - dtm_placeholders := map[string]DtmMultiTypeMap{} - temp_html_file := os.join_path(dtmi.template_folder, temp_html_fp) - html_mem := dtmi.get_cache(temp_html_n, temp_html_file, &dtm_placeholders) - assert html_mem.len > 10 - } + dtm_placeholders := map[string]DtmMultiTypeMap{} + temp_html_file := os.join_path(dtmi.template_folder, temp_html_fp) + html_mem := dtmi.get_cache(temp_html_n, temp_html_file, &dtm_placeholders) + assert html_mem.len > 10 } fn test_chandler_clear_specific_cache() { @@ -76,20 +74,18 @@ fn test_chandler_clear_specific_cache() { dtmi.stop_cache_handler() } dtmi.create_cache() - if !dtmi.abort_test { - lock dtmi.template_caches { - cache_file := os.join_path(dtmi.template_cache_folder, - '${dtmi.template_caches[0].name}_${dtmi.template_caches[0].checksum}.cache') - index, is_success := dtmi.chandler_clear_specific_cache(dtmi.template_caches[0].id) - assert is_success == true - assert index == 0 - cache_exist := os.exists(cache_file) - assert cache_exist == false - } + lock dtmi.template_caches { + cache_file := os.join_path(dtmi.template_cache_folder, + '${dtmi.template_caches[0].name}_${dtmi.template_caches[0].checksum}.cache') + index, is_success := dtmi.chandler_clear_specific_cache(dtmi.template_caches[0].id) + assert is_success == true + assert index == 0 + cache_exist := os.exists(cache_file) + assert cache_exist == false } } -fn test_handle_dtm_clock() { +fn test_cache_timestamp_is_initialized() { mut dtmi := init_dtm(true, max_size_data_in_memory) defer { dtmi.stop_cache_handler() @@ -98,30 +94,26 @@ fn test_handle_dtm_clock() { assert date_to_str.len > 10 } -fn test_cache_handler() { +fn test_process_rendered_cache_delete_request() { mut dtmi := init_dtm(true, max_size_data_in_memory) defer { dtmi.stop_cache_handler() } dtmi.create_cache() - if !dtmi.abort_test { - path_f := os.join_path(dtmi.template_folder, temp_html_fp) - lock dtmi.template_caches { - assert dtmi.template_caches[0].id == 1 - assert dtmi.template_caches[0].name == temp_html_n - assert dtmi.template_caches[0].path == path_f - } - dtmi.id_to_handlered = 1 - dtmi.ch_cache_handler <- TemplateCache{ - id: 1 - cache_request: .delete - } - dtmi.sync_cache() - if !dtmi.abort_test { - lock dtmi.template_caches { - assert dtmi.template_caches.len == 0 - } - } + path_f := os.join_path(dtmi.template_folder, temp_html_fp) + lock dtmi.template_caches { + assert dtmi.template_caches[0].id == 1 + assert dtmi.template_caches[0].name == temp_html_n + assert dtmi.template_caches[0].path == path_f + } + dtmi.id_to_handlered = 1 + dtmi.ch_cache_handler <- TemplateCache{ + id: 1 + cache_request: .delete + } + dtmi.sync_cache() + rlock dtmi.template_caches { + assert dtmi.template_caches.len == 0 } } diff --git a/vlib/x/templating/dtm/dynamic_template_manager_dtm2_bridge.v b/vlib/x/templating/dtm/dynamic_template_manager_dtm2_bridge.v new file mode 100644 index 00000000000000..dd05f9ef714955 --- /dev/null +++ b/vlib/x/templating/dtm/dynamic_template_manager_dtm2_bridge.v @@ -0,0 +1,65 @@ +module dtm + +import x.templating.dtm2 + +// new_dtm2_render_engine is the compatibility boundary between the historical +// DTM API and the DTM2 runtime template engine. DTM2 remains unaware +// of legacy cache concepts; it only parses templates and renders them. +fn new_dtm2_render_engine(template_dir string, compress_html bool) &dtm2.Manager { + return dtm2.initialize( + template_dir: template_dir + compress_html: compress_html + reload_modified_templates: true + ) +} + +fn (mut tm DynamicTemplateManager) expand_with_dtm2_render_engine(tmpl_path string, tmpl_var TemplateCacheParams) string { + validate_legacy_placeholder_sizes(tmpl_path, tmpl_var.placeholders) or { + eprintln(err.msg()) + return internat_server_error + } + if isnil(tm.render_engine) { + tm.render_engine = new_dtm2_render_engine(tm.template_folder, tm.compress_html) + } + placeholders := convert_legacy_placeholders(tmpl_var.placeholders) + return tm.render_engine.expand(tmpl_path, + placeholders: &placeholders + missing_placeholder_prefix: '$' + ) +} + +fn validate_legacy_placeholder_sizes(tmpl_path string, placeholders &map[string]DtmMultiTypeMap) ! { + for key, value in placeholders { + if key.len > max_placeholders_key_size { + return error('${message_signature_error} Length of placeholder key "${key}" exceeds the maximum allowed size for template content in file: ${tmpl_path}. Max allowed size: ${max_placeholders_key_size} characters.') + } + casted_value := legacy_placeholder_value_to_string(*value) + if casted_value.len > max_placeholders_value_size { + return error('${message_signature_error} Length of placeholder value for key "${key}" exceeds the maximum allowed size for template content in file: ${tmpl_path}. Max allowed size: ${max_placeholders_value_size} characters.') + } + } +} + +fn convert_legacy_placeholders(placeholders &map[string]DtmMultiTypeMap) map[string]string { + mut converted := map[string]string{} + for key, value in placeholders { + converted[key.clone()] = legacy_placeholder_value_to_string(*value).clone() + } + return converted +} + +fn legacy_placeholder_value_to_string(value DtmMultiTypeMap) string { + return match value { + f32 { value.str() } + f64 { value.str() } + i16 { value.str() } + i64 { value.str() } + i8 { value.str() } + int { value.str() } + string { value.clone() } + u16 { value.str() } + u32 { value.str() } + u64 { value.str() } + u8 { value.str() } + } +} diff --git a/vlib/x/templating/dtm/dynamic_template_manager_test.v b/vlib/x/templating/dtm/dynamic_template_manager_test.v index 28744d3569f636..f2cdf6b1471cc9 100644 --- a/vlib/x/templating/dtm/dynamic_template_manager_test.v +++ b/vlib/x/templating/dtm/dynamic_template_manager_test.v @@ -160,7 +160,7 @@ fn test_check_tmpl_and_placeholders_size() { } fn test_chandler_prevent_cache_duplicate_request() { - dtmi := init_dtm(false, 0) + mut dtmi := init_dtm(false, 0) temp_html_file := os.join_path(dtmi.template_folder, temp_html_fp) lock dtmi.template_caches { diff --git a/vlib/x/templating/dtm/escape_html_strings_in_templates.v b/vlib/x/templating/dtm/escape_html_strings_in_templates.v index fbe88a3640093e..6819c449472db5 100644 --- a/vlib/x/templating/dtm/escape_html_strings_in_templates.v +++ b/vlib/x/templating/dtm/escape_html_strings_in_templates.v @@ -1,7 +1,30 @@ module dtm -import encoding.html +import strings fn filter(s string) string { - return html.escape(s) + mut escaped := strings.new_builder(s.len) + for i := 0; i < s.len; i++ { + match s[i] { + `&` { + escaped.write_string('&') + } + `<` { + escaped.write_string('<') + } + `>` { + escaped.write_string('>') + } + `"` { + escaped.write_string('"') + } + `'` { + escaped.write_string(''') + } + else { + escaped.write_u8(s[i]) + } + } + } + return escaped.str() } diff --git a/vlib/x/templating/dtm/tmpl.v b/vlib/x/templating/dtm/tmpl.v index 1c8a91a2717c0c..fee8905d069ab5 100644 --- a/vlib/x/templating/dtm/tmpl.v +++ b/vlib/x/templating/dtm/tmpl.v @@ -79,6 +79,9 @@ fn is_html_open_tag(name string, s string) bool { fn replace_placeholders_with_data(line string, data &map[string]DtmMultiTypeMap, state State) string { mut rline := line + if data.len == 0 { + return rline + } mut need_include_html := false for key, value in data { @@ -104,12 +107,13 @@ fn replace_placeholders_with_data(line string, data &map[string]DtmMultiTypeMap, // Checks if the placeholder allows HTML inclusion if need_include_html { if state == State.html { - // Iterates over allowed HTML tags for inclusion + // Escape the whole value first, then restore only the explicit allow-list. + // This preserves the documented opt-in HTML behavior without allowing + // arbitrary raw tags through `_#includehtml`. + val_str = filter(value) for tag in allowed_tags { - // Escapes the HTML tag escaped_tag := filter(tag) - // Replaces the escaped tags with actual HTML tags in the value - val_str = value.replace(escaped_tag, tag) + val_str = val_str.replace(escaped_tag, tag) } } else { val_str = filter(value) @@ -124,10 +128,6 @@ fn replace_placeholders_with_data(line string, data &map[string]DtmMultiTypeMap, rline = rline.replace(placeholder, val_str) } - // If no output is found for the placeholder being processed, then the placeholder is escaped - if rline.contains('$') { - rline = filter(rline) - } return rline } @@ -136,9 +136,8 @@ fn insert_template_code(fn_name string, tmpl_str_start string, line string, data // HTML, may include `@var` // escaped by cgen, unless it's a `veb.RawHtml` string trailing_bs := tmpl_str_end + 'sb_${fn_name}.write_u8(92)\n' + tmpl_str_start - round1 := ['\\', '\\\\', r"'", "\\'", r'@', r'$'] - round2 := [r'$$', r'\@', r'.$', r'.@'] - mut rline := line.replace_each(round1).replace_each(round2) + mut rline := line.replace('\\', '\\\\').replace("'", "\\'").replace('@', '$') + rline = rline.replace(r'$$', r'\@').replace(r'.$', r'.@') comptime_call_str := rline.find_between('\${', '}') if comptime_call_str.contains("\\'") { rline = rline.replace(comptime_call_str, comptime_call_str.replace("\\'", r"'")) @@ -154,10 +153,11 @@ fn insert_template_code(fn_name string, tmpl_str_start string, line string, data // compile_file compiles the content of a file by the given path as a template fn compile_template_file(template_file string, fn_name string, data &map[string]DtmMultiTypeMap) string { - mut lines := os.read_lines(template_file) or { + template_content := os.read_file(template_file) or { eprintln('${message_signature_error} Template generator can not reading from ${template_file} file') return internat_server_error } + mut lines := template_content.split_into_lines() basepath := os.dir(template_file) diff --git a/vlib/x/templating/dtm2/README.md b/vlib/x/templating/dtm2/README.md new file mode 100644 index 00000000000000..7e2f682db3b134 --- /dev/null +++ b/vlib/x/templating/dtm2/README.md @@ -0,0 +1,365 @@ +# dtm2 - Dynamic Template Manager 2 + +`dtm2` is the modern runtime renderer for Dynamic Template Manager. +It keeps the original DTM idea: templates are normal files on disk, so they can +be edited without recompiling the application. + +The main change from `x.templating.dtm` is architectural. `dtm2` caches parsed +template trees, not rendered HTML responses. This keeps rendering fast while +removing the old async rendered-cache server from the hot path. + +## Quick Start + +Create a `templates/` folder in your application and put your templates inside +it. DTM2 supports `.html`, `.htm`, `.xml`, `.txt`, and `.text` by default. + +```v +import x.templating.dtm2 + +fn main() { + mut manager := dtm2.initialize( + template_dir: 'templates' + ) + placeholders := { + 'title': 'DTM2' + 'body': '<strong>escaped by default</strong>' + } + rendered := manager.expand('page.html', placeholders: &placeholders) + println(rendered) +} +``` + +Example template: + +```html +<!doctype html> +<html> + <head> + <title>@title + + +
@body
+ + +``` + +The rendered `@body` value is escaped by default. + +## Veb Example + +```v +import veb +import x.templating.dtm2 + +pub struct App { +pub mut: + templates &dtm2.Manager = unsafe { nil } +} + +pub struct Context { + veb.Context +} + +fn main() { + mut app := &App{ + templates: dtm2.initialize( + template_dir: 'templates' + ) + } + veb.run[App, Context](mut app, 18081) +} + +@['/'] +pub fn (mut app App) index(mut ctx Context) veb.Result { + placeholders := { + 'title': 'Home' + 'body': 'Hello from DTM2' + } + html := app.templates.expand('index.html', placeholders: &placeholders) + return ctx.html(html) +} +``` + +## Available Options + +### Manager options + +`dtm2.initialize()` accepts: + +- `template_dir` (**string**): root directory used for relative template paths. + If empty, `/templates` is used. +- `compress_html` (**bool**): enables a lightweight deterministic HTML + whitespace compressor. It is enabled by default. +- `reload_modified_templates` (**bool**): when enabled, DTM2 checks the source + template and included files before reusing a parsed template tree. It is + enabled by default. +- `extension_config_file` (**string**): optional JSON file containing extension + mappings. If empty, DTM2 automatically loads `dtm2_extensions.json` from the + configured `template_dir` when that file exists. + +Example: + +```v ignore +mut manager := dtm2.initialize( + template_dir: 'templates' + compress_html: true + reload_modified_templates: true +) +``` + +For maximum hot-path throughput in applications where templates are immutable +after startup, you can disable reload checks: + +```v ignore +mut manager := dtm2.initialize( + template_dir: 'templates' + reload_modified_templates: false +) +``` + +The recommended runtime model is one long-lived manager per application or +rendering context. Reusing the manager is what keeps parsed templates and path +resolution cached. + +### Template extensions + +DTM2 has two rendering modes: + +- `TemplateType.html`: HTML/XML-like output, with default escaping and optional + HTML compression. +- `TemplateType.text`: raw text output, also escaped by default. + +Default mappings: + +- HTML mode: `.html`, `.htm`, `.xml` +- Text mode: `.txt`, `.text` + +Project-specific extensions should be configured with a JSON file. + +Example `templates/dtm2_extensions.json`: + +```json +{ + "html": [".view", ".tmpl"], + "text": [".mail", ".md"] +} +``` + +If the file is named `dtm2_extensions.json` and is placed directly in the +configured `template_dir`, DTM2 loads it automatically: + +```v ignore +mut manager := dtm2.initialize( + template_dir: 'templates' +) +``` + +You can also point to an explicit config file: + +```v ignore +mut manager := dtm2.initialize( + template_dir: 'templates' + extension_config_file: 'config/my_dtm2_extensions.json' +) +``` + +DTM2 ships a default config example that can be copied into your own +`templates/` directory: + +```sh +vlib/x/templating/dtm2/dtm2_extensions.json +``` + +If the JSON file is absent, invalid, or contains invalid entries, DTM2 keeps the +built-in defaults and prints a warning for entries that cannot be registered. +JSON extension config files are limited to 64 KB and every extension is +validated before being registered. + +### Render options + +`manager.expand()` accepts: + +- `placeholders` (**&map[string]string**): values inserted in the template. +- `missing_placeholder_prefix` (**string**): prefix written when a placeholder + is missing. The default is `@`, preserving the original placeholder text. + +Example: + +```v ignore +placeholders := { + 'title': 'Example' +} +html := manager.expand('page.html', + placeholders: &placeholders + missing_placeholder_prefix: '@' +) +``` + +## The Placeholder System + +Template placeholders use the `@name` form. + +```html +

@title

+

@body

+``` + +In DTM2, placeholder values are strings: + +```v ignore +placeholders := { + 'title': 'Hello' + 'count': '42' +} +``` + +Values are escaped by default in both HTML and text templates. + +**Security note for custom non-HTML formats such as SQL:** DTM2 is a template +renderer, not a domain-specific sanitizer. By default, it escapes HTML-special +characters in placeholder values. It does not make SQL queries safe, does not +replace prepared statements, and does not validate business-specific formats. +If you add `.sql` or another sensitive extension through configuration, the +security of that generated content remains the responsibility of the +application. + +## Explicit HTML Inclusion + +The historical `_#includehtml` suffix is still supported for compatibility. +It allows a placeholder to include a restricted set of HTML tags in `.html` +templates. + +```v ignore +placeholders := { + 'body_#includehtml': '

allowed

' +} +html := manager.expand('page.html', placeholders: &placeholders) +``` + +The template still uses the normal placeholder name: + +```html +
@body
+``` + +DTM2 escapes the complete value first, then restores only allowed tags. This +preserves the old opt-in behavior without allowing arbitrary raw HTML through. +In `.txt` templates, HTML is always escaped. + +Allowed tags: + +```html +
,
,

,

,

,

,

,

,

,

, +
,
,
,
,

,

,
,
, , , +
    ,
,
    ,
,
  • ,
    • ,
    ,
    ,
    ,
    , +
    ,
    , , , ,
    , , , +, , , , , , , , +, , , , , , , , +, ,
    ,
    ,
    ,
    , +
    ,
    ,
    ,
    ,
    ,
    , +,
    ,
    , , , +, , , +``` + +## Includes + +Templates can include other templates with a simple line-level directive: + +```html +
    @include "partials/nav"
    +
    @body
    +``` + +Include paths are resolved relative to the current template. If no file +extension is provided, `.html` is added. The final resolved include path must +stay inside the manager `template_dir`; attempts to include files through `../` +or absolute paths outside that root fail. + +The same boundary applies to templates passed to `expand()`. Absolute template +paths are accepted only when they resolve inside `template_dir`. + +Included files are tracked as dependencies of the parsed template. When +`reload_modified_templates` is enabled, changing an included file invalidates +the cached parsed tree. + +## Backward Compatibility With DTM v1 + +Existing code that imports `x.templating.dtm` is kept source-compatible for the +migration period. The v1 facade is deprecated, but it now delegates rendering to +DTM2 internally. + +That means old code can continue to compile: + +```v ignore +import x.templating.dtm + +mut manager := dtm.initialize() +mut placeholders := map[string]dtm.DtmMultiTypeMap{} +placeholders['title'] = 'Legacy DTM' +placeholders['count'] = 7 +html := manager.expand('page.html', placeholders: &placeholders) +``` + +For new code, prefer importing `x.templating.dtm2` directly: + +```v ignore +import x.templating.dtm2 + +mut manager := dtm2.initialize(template_dir: 'templates') +placeholders := { + 'title': 'Modern DTM' + 'count': '7' +} +html := manager.expand('page.html', placeholders: &placeholders) +``` + +Migration notes: + +- Replace `import x.templating.dtm` with `import x.templating.dtm2`. +- Replace `DtmMultiTypeMap` placeholder maps with `map[string]string`. +- Convert numeric values to strings before rendering. +- Remove `stop_cache_handler()` calls; DTM2 does not start an async cache + server. +- Keep using `_#includehtml` only when HTML inclusion is intentional. + +## Design Notes + +DTM2 intentionally keeps rendering and rendered-output caching separate. + +The manager caches: + +- canonical template paths; +- parsed template trees; +- dependency metadata for root templates and includes. + +The manager does not cache rendered HTML responses. If a future rendered-cache +layer is needed, it should remain a small optional layer above DTM2 rather than +part of the parser/renderer core. + +## Benchmarks + +The local benchmark harness lives in: + +```sh +vlib/x/templating/dtm2/benchmarks/ +``` + +Run it from the repository root: + +```sh +vlib/x/templating/dtm2/benchmarks/run_dtm2_benchmark.sh +``` + +Useful options: + +- `DTM2_BENCH_MODE=prod|prod_o2|dev` +- `DTM2_BENCH_CASE=all|small_hot|small_cold|many_hot|many_cold|include_hot|include_cold|xml_hot|xml_cold` +- `DTM2_BENCH_ITERATIONS=50000` +- `DTM2_BENCH_COLD_ITERATIONS=500` +- `DTM2_BENCH_PLACEHOLDERS=50` +- `DTM2_BENCH_COMPRESS_HTML=true` +- `DTM2_BENCH_RELOAD_MODIFIED_TEMPLATES=false` +- `DTM2_BENCH_VALIDATE_EACH_ITERATION=false` + +Benchmark result directories are generated locally under +`vlib/x/templating/dtm2/benchmarks/results/` and should not be committed. diff --git a/vlib/x/templating/dtm2/benchmarks/dtm2_benchmark.v b/vlib/x/templating/dtm2/benchmarks/dtm2_benchmark.v new file mode 100644 index 00000000000000..e33bf586aeeec2 --- /dev/null +++ b/vlib/x/templating/dtm2/benchmarks/dtm2_benchmark.v @@ -0,0 +1,212 @@ +module main + +import os +import strings +import time +import x.templating.dtm2 + +const bench_root_name = 'dtm2_benchmark' + +struct BenchConfig { + case_name string + iterations int + cold_iterations int + placeholder_count int + compress_html bool + reload_modified_templates bool + validate_each_iteration bool +} + +fn main() { + config := BenchConfig{ + case_name: os.getenv('DTM2_BENCH_CASE') + iterations: env_int('DTM2_BENCH_ITERATIONS', 50000) + cold_iterations: env_int('DTM2_BENCH_COLD_ITERATIONS', 500) + placeholder_count: env_int('DTM2_BENCH_PLACEHOLDERS', 50) + compress_html: env_bool('DTM2_BENCH_COMPRESS_HTML', true) + reload_modified_templates: env_bool('DTM2_BENCH_RELOAD_MODIFIED_TEMPLATES', false) + validate_each_iteration: env_bool('DTM2_BENCH_VALIDATE_EACH_ITERATION', false) + } + root := os.join_path(os.vtmp_dir(), '${bench_root_name}_${os.getpid()}') + os.rmdir_all(root) or {} + setup_files(root, config)! + defer { + os.rmdir_all(root) or {} + } + + print_config(root, config) + small := small_placeholders() + many := many_placeholders(config.placeholder_count) + include := include_placeholders() + + if should_run(config, 'small_hot') { + bench_hot('small_hot', root, 'small.html', small, config.iterations, config) + } + if should_run(config, 'small_cold') { + bench_cold('small_cold', root, 'small.html', small, config.cold_iterations, config) + } + if should_run(config, 'many_hot') { + bench_hot('many_hot', root, 'many.html', many, config.iterations, config) + } + if should_run(config, 'many_cold') { + bench_cold('many_cold', root, 'many.html', many, config.cold_iterations, config) + } + if should_run(config, 'include_hot') { + bench_hot('include_hot', root, 'with_include.html', include, config.iterations, config) + } + if should_run(config, 'include_cold') { + bench_cold('include_cold', root, 'with_include.html', include, config.cold_iterations, + config) + } + if should_run(config, 'xml_hot') { + bench_hot('xml_hot', root, 'feed.xml', small, config.iterations, config) + } + if should_run(config, 'xml_cold') { + bench_cold('xml_cold', root, 'feed.xml', small, config.cold_iterations, config) + } +} + +fn print_config(root string, config BenchConfig) { + println('DTM2 benchmark') + println('root: ${root}') + println('case_name: ${if config.case_name == '' { 'all' } else { config.case_name }}') + println('iterations: ${config.iterations}') + println('cold_iterations: ${config.cold_iterations}') + println('placeholder_count: ${config.placeholder_count}') + println('compress_html: ${config.compress_html}') + println('reload_modified_templates: ${config.reload_modified_templates}') + println('validate_each_iteration: ${config.validate_each_iteration}') + println('') +} + +fn env_int(name string, default_value int) int { + raw := os.getenv(name) + if raw == '' { + return default_value + } + value := raw.int() + if value <= 0 { + return default_value + } + return value +} + +fn env_bool(name string, default_value bool) bool { + raw := os.getenv(name).to_lower() + if raw == '' { + return default_value + } + if raw in ['1', 'true', 'yes', 'on'] { + return true + } + if raw in ['0', 'false', 'no', 'off'] { + return false + } + return default_value +} + +fn should_run(config BenchConfig, label string) bool { + return config.case_name == '' || config.case_name == 'all' || config.case_name == label +} + +fn setup_files(root string, config BenchConfig) ! { + os.mkdir_all(os.join_path(root, 'partials'))! + os.write_file(os.join_path(root, 'small.html'), + '
    @title

    @body

    @count
    ')! + mut many := strings.new_builder(config.placeholder_count * 30) + many.writeln('
    ') + for i := 0; i < config.placeholder_count; i++ { + many.writeln('
    ${i}
    @p_${i}
    ') + } + many.writeln('
    ') + os.write_file(os.join_path(root, 'many.html'), many.str())! + os.write_file(os.join_path(root, 'partials', 'nav.html'), '')! + os.write_file(os.join_path(root, 'with_include.html'), + '
    @include "partials/nav"
    @body
    ')! + os.write_file(os.join_path(root, 'feed.xml'), + '@title@body@count')! +} + +fn small_placeholders() map[string]string { + mut placeholders := map[string]string{} + placeholders['title'] = 'Small'.clone() + placeholders['body'] = 'Body '.clone() + placeholders['count'] = '42' + return placeholders +} + +fn many_placeholders(count int) map[string]string { + mut placeholders := map[string]string{} + for i := 0; i < count; i++ { + placeholders['p_${i}'] = 'value ${i}'.clone() + } + return placeholders +} + +fn include_placeholders() map[string]string { + mut placeholders := map[string]string{} + placeholders['title'] = 'Menu'.clone() + placeholders['body'] = 'Body '.clone() + return placeholders +} + +fn bench_hot(label string, root string, template_path string, placeholders map[string]string, iterations int, config BenchConfig) { + mut manager := new_manager(root, config) + expected := manager.expand(template_path, placeholders: &placeholders).clone() + mut last_len := expected.len + sw := time.new_stopwatch() + for i in 0 .. iterations { + rendered := manager.expand(template_path, placeholders: &placeholders) + if config.validate_each_iteration { + validate_rendered(label, i, expected, rendered) + } + last_len = rendered.len + } + elapsed := sw.elapsed() + validate_rendered(label, iterations, expected, manager.expand(template_path, + placeholders: &placeholders + )) + print_result(label, iterations, elapsed, last_len, manager.compiled_template_count()) +} + +fn bench_cold(label string, root string, template_path string, placeholders map[string]string, iterations int, config BenchConfig) { + mut expected_manager := new_manager(root, config) + expected := expected_manager.expand(template_path, placeholders: &placeholders).clone() + mut last_len := expected.len + sw := time.new_stopwatch() + for i in 0 .. iterations { + mut manager := new_manager(root, config) + rendered := manager.expand(template_path, placeholders: &placeholders) + if config.validate_each_iteration { + validate_rendered(label, i, expected, rendered) + } + last_len = rendered.len + } + elapsed := sw.elapsed() + mut final_manager := new_manager(root, config) + validate_rendered(label, iterations, expected, final_manager.expand(template_path, + placeholders: &placeholders + )) + print_result(label, iterations, elapsed, last_len, final_manager.compiled_template_count()) +} + +fn new_manager(root string, config BenchConfig) &dtm2.Manager { + return dtm2.initialize( + template_dir: root + compress_html: config.compress_html + reload_modified_templates: config.reload_modified_templates + ) +} + +fn validate_rendered(label string, iteration int, expected string, rendered string) { + if rendered != expected { + eprintln('${label} invalid output at iteration ${iteration}: expected_len=${expected.len} actual_len=${rendered.len}') + exit(1) + } +} + +fn print_result(label string, iterations int, elapsed time.Duration, last_len int, compiled_count int) { + ns_per_op := if iterations > 0 { elapsed.nanoseconds() / i64(iterations) } else { i64(0) } + ops_per_sec := if ns_per_op > 0 { 1_000_000_000.0 / f64(ns_per_op) } else { 0.0 } + println('${label:-16} iterations=${iterations:8} total_ms=${elapsed.milliseconds():8} ns_per_op=${ns_per_op:10} ops_per_sec=${ops_per_sec:10.1f} last_len=${last_len:6} compiled_templates=${compiled_count}') +} diff --git a/vlib/x/templating/dtm2/benchmarks/run_dtm2_benchmark.sh b/vlib/x/templating/dtm2/benchmarks/run_dtm2_benchmark.sh new file mode 100755 index 00000000000000..1fb2ebf09c012a --- /dev/null +++ b/vlib/x/templating/dtm2/benchmarks/run_dtm2_benchmark.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/../../../../.." && pwd)" +cd "${repo_root}" + +timestamp="$(date +%Y-%m-%d_%H%M%S)" +results_dir="${DTM2_BENCH_RESULTS_DIR:-vlib/x/templating/dtm2/benchmarks/results/${timestamp}}" +bench_source="vlib/x/templating/dtm2/benchmarks/dtm2_benchmark.v" +bench_bin="/tmp/dtm2_benchmark_${timestamp}_$$" +bench_mode="${DTM2_BENCH_MODE:-prod}" + +mkdir -p "${results_dir}" +trap 'rm -f "${bench_bin}"' EXIT + +case "${bench_mode}" in + dev) + build_cmd=(./v -o "${bench_bin}" "${bench_source}") + ;; + prod) + build_cmd=(./v -prod -o "${bench_bin}" "${bench_source}") + ;; + prod_o2) + build_cmd=(./v -prod -no-prod-options -cflags -O2 -o "${bench_bin}" "${bench_source}") + ;; + *) + echo "unsupported DTM2_BENCH_MODE=${bench_mode}; use dev, prod, or prod_o2" >&2 + exit 2 + ;; +esac + +{ + echo "date: $(date -Iseconds)" + echo "v_version: $(./v --version)" + echo "kernel: $(uname -srmo)" + echo "cpu_model: $(awk -F': ' '/model name/{print $2; exit}' /proc/cpuinfo)" + echo "logical_cpu_count: $(nproc)" + echo "memory_total: $(awk '/MemTotal/{print $2 " " $3}' /proc/meminfo)" + echo "benchmark_mode: ${bench_mode}" + echo "benchmark_source: ${bench_source}" + echo "DTM2_BENCH_CASE: ${DTM2_BENCH_CASE:-all}" + echo "DTM2_BENCH_ITERATIONS: ${DTM2_BENCH_ITERATIONS:-50000}" + echo "DTM2_BENCH_COLD_ITERATIONS: ${DTM2_BENCH_COLD_ITERATIONS:-500}" + echo "DTM2_BENCH_PLACEHOLDERS: ${DTM2_BENCH_PLACEHOLDERS:-50}" + echo "DTM2_BENCH_COMPRESS_HTML: ${DTM2_BENCH_COMPRESS_HTML:-true}" + echo "DTM2_BENCH_RELOAD_MODIFIED_TEMPLATES: ${DTM2_BENCH_RELOAD_MODIFIED_TEMPLATES:-false}" + echo "DTM2_BENCH_VALIDATE_EACH_ITERATION: ${DTM2_BENCH_VALIDATE_EACH_ITERATION:-false}" +} > "${results_dir}/environment.txt" + +"${build_cmd[@]}" > "${results_dir}/build.log" 2>&1 + +{ + cat "${results_dir}/environment.txt" + echo + /usr/bin/time -v "${bench_bin}" +} 2>&1 | tee "${results_dir}/benchmark.log" diff --git a/vlib/x/templating/dtm2/dtm2_extensions.json b/vlib/x/templating/dtm2/dtm2_extensions.json new file mode 100644 index 00000000000000..ae88663467a820 --- /dev/null +++ b/vlib/x/templating/dtm2/dtm2_extensions.json @@ -0,0 +1,4 @@ +{ + "html": [".html", ".htm", ".xml"], + "text": [".txt", ".text"] +} diff --git a/vlib/x/templating/dtm2/dynamic_template_manager2.v b/vlib/x/templating/dtm2/dynamic_template_manager2.v new file mode 100644 index 00000000000000..fc76a37931ec41 --- /dev/null +++ b/vlib/x/templating/dtm2/dynamic_template_manager2.v @@ -0,0 +1,750 @@ +module dtm2 + +import os +import json +import strings + +// dtm2 is the modern runtime renderer for Dynamic Template Manager. +// It keeps DTM dynamic, meaning templates can still be edited on disk without +// recompiling the application, while moving the hot path to a parsed-template +// cache instead of a rendered-output cache. File extensions are configurable, +// but every template ultimately renders in either HTML mode or text mode. +// +// The intended runtime model is one long-lived Manager per application or +// rendering context. Tests and benchmarks may create short-lived managers, but +// normal applications should reuse a manager so parsed templates and resolved +// paths stay hot. +const message_signature = '[Dynamic Template Manager 2]' +const internal_server_error = 'Internal Server Error' +const include_html_key_suffix = '_#includehtml' +const max_include_depth = 32 +const max_extension_config_size = 64 * 1024 +const default_extension_config_filename = 'dtm2_extensions.json' +const segment_instruction_size = 9 +const segment_kind_text = u8(0) +const segment_kind_placeholder = u8(1) + +// These tags are the compatibility allow-list for the historical `_#includehtml` +// placeholder suffix. Any other tag is still escaped. +const allowed_html_tags = ['
    ', '
    ', '

    ', '

    ', '

    ', '

    ', '

    ', '

    ', + '

    ', '

    ', '
    ', '
    ', '
    ', '
    ', '

    ', '

    ', '
    ', '
    ', '', + '', '
      ', '
    ', '
      ', '
    ', '
  • ', '
    • ', '
    ', '
    ', '
    ', '
    ', + '
    ', '
    ', '', '', '', '
    ', '', '', '', + '', '', '', '', '', '', '', '', '', + '', '', '', '', '', '', '', '', + '
    ', '
    ', '
    ', '
    ', '
    ', '
    ', '
    ', '
    ', + '
    ', '
    ', '', '
    ', '
    ', '', + '', '', '', '', '']! + +pub enum TemplateType { + html + text +} + +// ExtensionConfig is the JSON representation accepted by +// `ManagerParams.extension_config_file`. +// +// Example: +// +// ```json +// { +// "html": [".html", ".htm", ".xml", ".view"], +// "text": [".txt", ".mail"] +// } +// ``` +pub struct ExtensionConfig { +pub: + html []string + text []string +} + +// TemplateDependency records the file metadata that decides whether a parsed +// template is still fresh. Includes are tracked as dependencies of the parent +// template, so editing a partial can invalidate the compiled parent tree. +struct TemplateDependency { + path string + modified_at i64 + size u64 +} + +// CompiledTemplate is the parsed representation stored by Manager. +// +// It intentionally stores a compact instruction string instead of slices of +// segment structs. The instructions contain offsets into the owned `content` +// string, which keeps the cache compact and avoids keeping fragile string-slice +// graphs alive across many -prod renders. +@[heap] +struct CompiledTemplate { + // Rendering mode inferred from the source file extension. + template_type TemplateType + // Full template content after static @include expansion. + content string + // Compact metadata used to decide whether this compiled tree is stale. + dependency_signature string + // Binary instruction stream: one fixed-size record per text/placeholder segment. + instructions string + // Number of text and placeholder records in `instructions`. + segment_count int + // Static-text size hint used to preallocate the render buffer. + estimated_size int +} + +// Manager owns the runtime state of a dtm2 renderer. +// +// It caches resolved template paths and parsed template trees, but never caches +// rendered HTML responses. This keeps the new engine deterministic and leaves +// legacy rendered-cache compatibility in `x.templating.dtm`. +@[heap] +pub struct Manager { +mut: + // Base directory for relative template paths. + template_dir string + // Enables the lightweight deterministic HTML whitespace compressor. + compress_html bool + // When true, source and include files are stat-checked before cache reuse. + reload_modified_templates bool + // Maps caller-provided template paths to canonical source paths. + resolved_template_paths map[string]string + // Parsed-template cache keyed by canonical source path. + compiled_templates map[string]&CompiledTemplate + // Extension-to-render-mode table. Users can extend or override it through + // ManagerParams without changing the parser or renderer. + template_extensions map[string]TemplateType +} + +// ManagerParams configure a dtm2 Manager. +@[params] +pub struct ManagerParams { +pub: + // Root directory used when `expand()` receives a relative template path. + // If empty, `/templates` is used. + template_dir string + // Compresses HTML output by removing newlines/tabs and redundant spaces. + compress_html bool = true + // Re-check source files and includes before reusing a parsed template tree. + // Disable it for maximum hot-path throughput when templates are immutable. + reload_modified_templates bool = true + // Additional or overriding extension mappings. + // Default mappings are: `.html`, `.htm`, `.xml` => HTML mode and + // `.txt`, `.text` => text mode. + template_extensions map[string]TemplateType + // Optional JSON file containing extension mappings. It is merged after the + // default mappings and before `template_extensions`, so explicit code + // configuration wins over the file. If empty, DTM2 tries to load + // `/dtm2_extensions.json` when that file exists. + extension_config_file string +} + +// RenderParams configure one render call. +@[params] +pub struct RenderParams { +pub: + // Placeholder values keyed by their template name without the `@` prefix. + // Values are escaped by default. + placeholders &map[string]string = &map[string]string{} + // Prefix written back when a placeholder is missing. The default preserves + // the original `@placeholder` text. + missing_placeholder_prefix string = '@' +} + +// initialize creates a dynamic template manager rooted at `params.template_dir`. +// +// The returned manager should normally be kept and reused. Reusing it is what +// gives dtm2 its parsed-template and path-resolution cache benefits. +pub fn initialize(params ManagerParams) &Manager { + raw_template_dir := if params.template_dir == '' { + os.join_path(os.dir(os.executable()), 'templates') + } else { + params.template_dir + } + template_dir := canonical_template_dir(raw_template_dir) + template_extensions := build_template_extensions(params, template_dir) + return &Manager{ + template_dir: template_dir.clone() + compress_html: params.compress_html + reload_modified_templates: params.reload_modified_templates + resolved_template_paths: map[string]string{} + compiled_templates: map[string]&CompiledTemplate{} + template_extensions: template_extensions + } +} + +fn canonical_template_dir(template_dir string) string { + return os.real_path(template_dir) +} + +fn build_template_extensions(params ManagerParams, template_dir string) map[string]TemplateType { + mut extensions := default_template_extensions() + config_path := extension_config_path(params, template_dir) + if config_path != '' { + file_extensions := read_extension_config_file(config_path) or { + eprintln('${message_signature} ${err.msg()}') + map[string]TemplateType{} + } + merge_template_extensions(mut extensions, file_extensions) + } + merge_template_extensions(mut extensions, params.template_extensions) + return extensions +} + +fn extension_config_path(params ManagerParams, template_dir string) string { + if params.extension_config_file != '' { + return params.extension_config_file + } + default_path := os.join_path(template_dir, default_extension_config_filename) + if os.exists(default_path) { + return default_path + } + return '' +} + +fn default_template_extensions() map[string]TemplateType { + return { + '.html': .html + '.htm': .html + '.xml': .html + '.txt': .text + '.text': .text + } +} + +fn read_extension_config_file(config_path string) !map[string]TemplateType { + if !os.exists(config_path) { + return error('extension config file "${config_path}" not found') + } + if os.is_dir(config_path) { + return error('extension config path "${config_path}" is a directory') + } + config_stat := os.stat(config_path)! + if config_stat.size > u64(max_extension_config_size) { + return error('extension config file "${config_path}" is larger than ${max_extension_config_size} bytes') + } + raw_config := os.read_file(config_path)! + config := json.decode(ExtensionConfig, raw_config)! + mut extensions := map[string]TemplateType{} + merge_template_extension_list(mut extensions, config.html, .html) + merge_template_extension_list(mut extensions, config.text, .text) + return extensions +} + +fn merge_template_extensions(mut target map[string]TemplateType, source map[string]TemplateType) { + for ext, template_type in source { + normalized := validate_template_extension(ext) or { + eprintln('${message_signature} ${err.msg()}') + continue + } + target[normalized] = template_type + } +} + +fn merge_template_extension_list(mut target map[string]TemplateType, extensions []string, template_type TemplateType) { + for ext in extensions { + normalized := validate_template_extension(ext) or { + eprintln('${message_signature} ${err.msg()}') + continue + } + target[normalized] = template_type + } +} + +fn normalize_template_extension(ext string) string { + trimmed := ext.trim_space().to_lower() + if trimmed == '' { + return '' + } + if trimmed.starts_with('.') { + return trimmed + } + return '.${trimmed}' +} + +fn validate_template_extension(ext string) !string { + normalized := normalize_template_extension(ext) + if normalized.len < 2 { + return error('ignoring invalid template extension "${ext}"') + } + for i := 1; i < normalized.len; i++ { + c := normalized[i] + if (c >= `a` && c <= `z`) || (c >= `0` && c <= `9`) || c == `_` || c == `-` || c == `.` { + continue + } + return error('ignoring invalid template extension "${ext}"') + } + return normalized +} + +// compiled_template_count is intentionally exposed for tests and diagnostics. In +// dtm2 it counts parsed template trees currently held by the manager. +pub fn (m &Manager) compiled_template_count() int { + return m.compiled_templates.len +} + +// expand renders `template_path` with the provided placeholders. +// +// `template_path` can be absolute or relative to the manager's template +// directory. Supported extensions come from the manager extension table. +// Errors are reported to stderr and return the legacy-compatible +// `Internal Server Error` string. +pub fn (mut m Manager) expand(template_path string, params RenderParams) string { + source_path := m.cached_template_path(template_path) or { + eprintln('${message_signature} ${err.msg()}') + return internal_server_error + } + compiled := m.compiled_template_for_path(source_path) or { + eprintln('${message_signature} ${err.msg()}') + return internal_server_error + } + if m.compress_html && compiled.template_type == .html { + rendered := render_compiled_template(compiled, params) + compressed := compress_html_output(rendered) + return compressed.clone() + } + rendered := render_compiled_template(compiled, params) + return rendered.clone() +} + +// cached_template_path resolves caller paths once and then reuses the canonical +// filesystem path. This removes repeated path normalization from the hot render +// loop while still keeping template content reload decisions separate. +fn (mut m Manager) cached_template_path(template_path string) !string { + if source_path := m.resolved_template_paths[template_path] { + return source_path + } + source_path := m.resolve_template_path(template_path)! + m.resolved_template_paths[template_path] = source_path + return source_path +} + +// compiled_template_for_path returns the parsed tree for a canonical source +// path. When reload checks are enabled, dependency metadata decides whether the +// tree can be reused or must be rebuilt from disk. +fn (mut m Manager) compiled_template_for_path(source_path string) !&CompiledTemplate { + if compiled := m.compiled_templates[source_path] { + if !m.reload_modified_templates || compiled.dependencies_are_fresh() { + return compiled + } + } + compiled := compile_template_from_file(source_path, m.template_dir, m.template_extensions)! + m.compiled_templates[source_path] = compiled + return compiled +} + +// compile_template_from_file reads a template, expands static includes, parses +// placeholders, and stores dependency metadata for future invalidation. +fn compile_template_from_file(source_path string, template_root string, template_extensions map[string]TemplateType) !&CompiledTemplate { + template_type := template_type_from_path(source_path, template_extensions)! + raw_content, dependencies := read_template_with_includes(source_path, template_root, 0)! + instructions, estimated_size, segment_count := parse_segments(raw_content) + dependency_signature := encode_dependencies(dependencies) + return &CompiledTemplate{ + template_type: template_type + content: copy_string(raw_content) + dependency_signature: copy_string(dependency_signature) + instructions: copy_string(instructions) + segment_count: segment_count + estimated_size: estimated_size + } +} + +// copy_string forces cached strings to own their memory. Cached templates live +// beyond the stack frame that built them, so the cache should not retain +// accidental views into temporary buffers. +fn copy_string(value string) string { + mut bytes := []u8{len: value.len} + for i := 0; i < value.len; i++ { + bytes[i] = value[i] + } + copied := bytes.bytestr() + return copied.clone() +} + +// dependencies_are_fresh checks root and included files. Both modification time +// and file size are used so same-second rewrites still invalidate reliably. +fn (compiled &CompiledTemplate) dependencies_are_fresh() bool { + signature := compiled.dependency_signature + mut offset := 0 + for offset < signature.len { + first_sep := index_byte_from(signature, `|`, offset) or { return false } + second_sep := index_byte_from(signature, `|`, first_sep + 1) or { return false } + line_end := index_byte_from(signature, `\n`, second_sep + 1) or { signature.len } + size := signature[first_sep + 1..second_sep].i64() + if size < 0 { + return false + } + path := signature[second_sep + 1..line_end] + stat := os.stat(path) or { return false } + if stat.mtime != signature[offset..first_sep].i64() || stat.size != u64(size) { + return false + } + offset = line_end + 1 + } + return true +} + +fn index_byte_from(value string, needle u8, start int) ?int { + for i := start; i < value.len; i++ { + if value[i] == needle { + return i + } + } + return none +} + +fn encode_dependencies(dependencies []TemplateDependency) string { + mut out := strings.new_builder(dependencies.len * 64) + for dependency in dependencies { + out.writeln('${dependency.modified_at}|${dependency.size}|${dependency.path}') + } + encoded := out.str() + return encoded.clone() +} + +// resolve_template_path converts absolute or manager-relative template names to +// canonical paths. The canonical path is used as the compiled-template cache key. +fn (m &Manager) resolve_template_path(template_path string) !string { + source_path := if os.is_abs_path(template_path) { + template_path + } else { + os.join_path(m.template_dir, template_path) + } + if !os.exists(source_path) { + return error('template "${source_path}" not found') + } + canonical_path := os.real_path(source_path) + ensure_path_inside_template_dir(canonical_path, m.template_dir)! + return canonical_path +} + +fn ensure_path_inside_template_dir(path string, template_root string) ! { + root := path_without_trailing_separator(template_root) + candidate := path_without_trailing_separator(path) + if candidate == root || candidate.starts_with(root + os.path_separator) { + return + } + return error('template "${path}" is outside template directory "${template_root}"') +} + +fn path_without_trailing_separator(path string) string { + if path.len > os.path_separator.len && path.ends_with(os.path_separator) { + return path[..path.len - os.path_separator.len] + } + return path +} + +// template_type_from_path keeps rendering rules extension-driven and explicit. +fn template_type_from_path(source_path string, template_extensions map[string]TemplateType) !TemplateType { + ext := normalize_template_extension(os.file_ext(source_path)) + if template_type := template_extensions[ext] { + return template_type + } + return error('template "${source_path}" uses unsupported extension "${ext}"') +} + +// read_template_with_includes expands simple line-level `@include "path"` +// directives before parsing placeholders. Includes are part of the parsed tree +// and are tracked as dependencies for reload checks. +fn read_template_with_includes(source_path string, template_root string, depth int) !(string, []TemplateDependency) { + if depth > max_include_depth { + return error('maximum @include depth exceeded while reading "${source_path}"') + } + content := os.read_file(source_path)! + base_dir := os.dir(source_path) + mut out := strings.new_builder(content.len) + mut dependencies := []TemplateDependency{cap: 4} + dependencies << template_dependency(source_path)! + lines := content.split_into_lines() + for line in lines { + expanded_line, line_dependencies := expand_include_directives(line, base_dir, + template_root, depth)! + out.write_string(expanded_line) + out.write_u8(`\n`) + dependencies << line_dependencies + } + rendered := out.str() + return rendered.clone(), dependencies +} + +fn template_dependency(source_path string) !TemplateDependency { + stat := os.stat(source_path)! + return TemplateDependency{ + path: source_path.clone() + modified_at: stat.mtime + size: stat.size + } +} + +fn expand_include_directives(line string, base_dir string, template_root string, depth int) !(string, []TemplateDependency) { + mut out := strings.new_builder(line.len) + mut dependencies := []TemplateDependency{} + mut offset := 0 + for { + rel_pos := line[offset..].index('@include ') or { + out.write_string(line[offset..]) + break + } + pos := offset + rel_pos + target, end_pos := include_target_from_line_at(line, pos) or { + out.write_string(line[offset..pos + '@include '.len]) + offset = pos + '@include '.len + continue + } + out.write_string(line[offset..pos]) + include_path := resolve_include_path(base_dir, target, template_root)! + next_depth := depth + 1 + included, include_dependencies := read_template_with_includes(include_path, template_root, + next_depth)! + out.write_string(included.trim_right('\n')) + dependencies << include_dependencies + offset = end_pos + } + expanded := out.str() + return expanded.clone(), dependencies +} + +fn include_target_from_line_at(line string, pos int) ?(string, int) { + mut cursor := pos + '@include '.len + for cursor < line.len && line[cursor].is_space() { + cursor++ + } + if cursor >= line.len { + return none + } + quote := line[cursor] + if quote != `'` && quote != `"` { + return none + } + start := cursor + 1 + for cursor = start; cursor < line.len; cursor++ { + if line[cursor] == quote { + return line[start..cursor], cursor + 1 + } + } + return none +} + +fn resolve_include_path(base_dir string, include_target string, template_root string) !string { + mut target := include_target + if os.file_ext(target) == '' { + target += '.html' + } + source_path := if os.is_abs_path(target) { + target + } else { + os.join_path(base_dir, target) + } + if !os.exists(source_path) { + return error('included template "${source_path}" not found') + } + include_path := os.real_path(source_path) + ensure_path_inside_template_dir(include_path, template_root)! + return include_path +} + +// parse_segments converts template content to a compact instruction stream. +// Each instruction stores a segment kind plus offset/length into `content`. +// Rendering can then walk the stream without reparsing placeholder syntax. +fn parse_segments(content string) (string, int, int) { + mut instructions := []u8{cap: segment_instruction_size * 16} + mut text_start := 0 + mut estimated_size := 0 + mut segment_count := 0 + mut i := 0 + for i < content.len { + if content[i] != `@` || i + 1 >= content.len || !is_placeholder_char(content[i + 1]) { + i++ + continue + } + if i > text_start { + append_segment_instruction(mut instructions, segment_kind_text, text_start, + i - text_start) + segment_count++ + estimated_size += i - text_start + } + mut end := i + 1 + for end < content.len && is_placeholder_char(content[end]) { + end++ + } + append_segment_instruction(mut instructions, segment_kind_placeholder, i + 1, end - i - 1) + segment_count++ + i = end + text_start = end + } + if text_start < content.len { + append_segment_instruction(mut instructions, segment_kind_text, text_start, + content.len - text_start) + segment_count++ + estimated_size += content.len - text_start + } + encoded := instructions.bytestr() + return encoded.clone(), estimated_size, segment_count +} + +fn append_segment_instruction(mut instructions []u8, kind u8, start int, len int) { + instructions << kind + append_u32(mut instructions, start) + append_u32(mut instructions, len) +} + +fn append_u32(mut data []u8, value int) { + encoded := u32(value) + data << u8(encoded & 0xff) + data << u8((encoded >> 8) & 0xff) + data << u8((encoded >> 16) & 0xff) + data << u8((encoded >> 24) & 0xff) +} + +fn read_u32(data string, offset int) int { + b0 := u32(data[offset]) + b1 := u32(data[offset + 1]) << 8 + b2 := u32(data[offset + 2]) << 16 + b3 := u32(data[offset + 3]) << 24 + value := b0 | b1 | b2 | b3 + return int(value) +} + +fn is_placeholder_char(c u8) bool { + return (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`) || (c >= `0` && c <= `9`) || c == `_` +} + +// render_compiled_template walks the parsed instruction stream and writes the +// final output. Placeholder values are rendered only for this call; the manager +// never stores caller data after `expand()` returns. +fn render_compiled_template(compiled &CompiledTemplate, params RenderParams) string { + mut out := strings.new_builder(estimate_render_size(compiled)) + mut offset := 0 + for offset + segment_instruction_size <= compiled.instructions.len { + kind := compiled.instructions[offset] + start := read_u32(compiled.instructions, offset + 1) + len := read_u32(compiled.instructions, offset + 5) + offset += segment_instruction_size + if kind == segment_kind_text { + text := compiled.segment_text(start, len) + out.write_string(text) + continue + } + if kind == segment_kind_placeholder { + name := compiled.segment_text(start, len) + if write_placeholder_value(mut out, name, params.placeholders, compiled.template_type) { + continue + } + out.write_string(params.missing_placeholder_prefix) + out.write_string(name) + } + } + rendered := out.str() + return rendered.clone() +} + +fn (compiled &CompiledTemplate) segment_text(start int, len int) string { + end := start + len + if start < 0 || len < 0 || end > compiled.content.len { + return '' + } + return compiled.content[start..end] +} + +// write_placeholder_value resolves the normal placeholder name first, then the +// historical `_#includehtml` alias. The alias is accepted for compatibility but +// still passes through the allow-list based HTML escaping path. +fn write_placeholder_value(mut out strings.Builder, name string, placeholders &map[string]string, template_type TemplateType) bool { + unsafe { + if raw_value := placeholders[name] { + rendered_value := render_value(raw_value, name.ends_with(include_html_key_suffix), + template_type) + out.write_string(rendered_value) + return true + } + include_html_name := name + include_html_key_suffix + if raw_html := placeholders[include_html_name] { + rendered_html := render_value(raw_html, true, template_type) + out.write_string(rendered_html) + return true + } + } + return false +} + +// estimate_render_size gives strings.Builder enough capacity for common +// placeholder expansion without making a second pre-render pass. +fn estimate_render_size(compiled &CompiledTemplate) int { + return compiled.estimated_size + 1024 + (compiled.segment_count * 64) +} + +// render_value applies DTM's safety default: values are escaped unless the +// caller explicitly uses the include-html convention in an HTML template. +fn render_value(raw string, allow_html bool, template_type TemplateType) string { + if allow_html && template_type == .html { + return escape_with_allowed_html(raw) + } + return escape_html(raw) +} + +// escape_with_allowed_html escapes the full value first, then restores only the +// supported tags. This preserves the legacy opt-in behavior without allowing +// arbitrary raw HTML through. +fn escape_with_allowed_html(value string) string { + mut escaped := escape_html(value) + for tag in allowed_html_tags { + escaped = escaped.replace(escape_html(tag), tag) + } + return escaped +} + +// escape_html is a local deterministic HTML escape helper. It avoids depending +// on heavier generic replacement paths in the hot render loop. +fn escape_html(value string) string { + mut escaped := strings.new_builder(value.len) + for i := 0; i < value.len; i++ { + match value[i] { + `&` { + escaped.write_string('&') + } + `<` { + escaped.write_string('<') + } + `>` { + escaped.write_string('>') + } + `"` { + escaped.write_string('"') + } + `'` { + escaped.write_string(''') + } + else { + escaped.write_u8(value[i]) + } + } + } + escaped_value := escaped.str() + return escaped_value.clone() +} + +// compress_html_output performs a small, predictable whitespace compression +// pass for HTML templates. It intentionally avoids regex work in the hot path. +fn compress_html_output(html string) string { + mut result := strings.new_builder(html.len) + mut pending_space := false + mut last_written := u8(0) + for i := 0; i < html.len; i++ { + c := html[i] + if c == `\n` || c == `\t` { + continue + } + if c == ` ` { + pending_space = true + continue + } + if pending_space { + if !(last_written == `>` && c == `<`) { + result.write_u8(` `) + last_written = ` ` + } + pending_space = false + } + result.write_u8(c) + last_written = c + } + compressed := result.str() + return compressed.clone() +} diff --git a/vlib/x/templating/dtm2/dynamic_template_manager2_test.v b/vlib/x/templating/dtm2/dynamic_template_manager2_test.v new file mode 100644 index 00000000000000..ee1ebbeb795b34 --- /dev/null +++ b/vlib/x/templating/dtm2/dynamic_template_manager2_test.v @@ -0,0 +1,306 @@ +module dtm2 + +import os +import time + +const test_root_name = 'dtm2_test' + +fn dtm2_test_root() string { + return os.join_path(os.vtmp_dir(), '${test_root_name}_${os.getpid()}') +} + +fn dtm2_outside_template_path() string { + return os.join_path(os.vtmp_dir(), '${test_root_name}_${os.getpid()}_outside.html') +} + +fn testsuite_begin() { + root := dtm2_test_root() + os.rmdir_all(root) or {} + os.mkdir_all(os.join_path(root, 'partials'))! + outside_path := dtm2_outside_template_path() + os.rm(outside_path) or {} + os.write_file(outside_path, '

    @title outside

    ')! + os.write_file(os.join_path(root, 'page.html'), '
    @title @body @missing
    ')! + os.write_file(os.join_path(root, 'invalid.css'), 'body { color: red; }')! + os.write_file(os.join_path(root, 'raw.txt'), 'Raw: @body')! + os.write_file(os.join_path(root, 'with_include.html'), + '
    @include "partials/nav"
    @title
    ')! + os.write_file(os.join_path(root, 'partials', 'nav.html'), '')! + os.write_file(os.join_path(root, 'include_outside_relative.html'), + '@include "../${os.base(outside_path)}"')! + os.write_file(os.join_path(root, 'include_outside_absolute.html'), '@include "${outside_path}"')! + os.write_file(os.join_path(root, 'reload.html'), '

    @title

    ')! + os.write_file(os.join_path(root, 'pinned.html'), '@title')! + os.write_file(os.join_path(root, 'path_alias.html'), '
    @title
    ')! + os.write_file(os.join_path(root, 'prefix.html'), '

    @unknown

    ')! + os.write_file(os.join_path(root, 'recursive.html'), '@include "recursive"')! + os.write_file(os.join_path(root, 'compress.html'), '
    \n @title\n
    ')! + os.write_file(os.join_path(root, 'feed.xml'), '@title')! + os.write_file(os.join_path(root, 'custom.page'), '
    @title
    ')! + os.write_file(os.join_path(root, 'custom.note'), 'Note: @body')! + os.write_file(os.join_path(root, 'custom.view'), '
    @title
    ')! + os.write_file(os.join_path(root, 'custom.mail'), 'Subject: @title')! + os.write_file(os.join_path(root, 'auto.fragment'), '')! + os.write_file(os.join_path(root, 'auto.message'), 'Auto: @title')! + os.write_file(os.join_path(root, 'bad.path'), '

    @title

    ')! + os.write_file(os.join_path(root, 'extensions.json'), '{"html":[".view"],"text":["mail"]}')! + os.write_file(os.join_path(root, default_extension_config_filename), + '{"html":[".fragment"],"text":[".message"]}')! + os.write_file(os.join_path(root, 'invalid_extensions.json'), + '{"html":["../bad"],"text":["bad/value",".message"]}')! +} + +fn testsuite_end() { + os.rmdir_all(dtm2_test_root()) or {} + os.rm(dtm2_outside_template_path()) or {} +} + +fn new_test_manager(compress_html bool) &Manager { + return initialize( + template_dir: dtm2_test_root() + compress_html: compress_html + ) +} + +fn test_render_placeholders_escape_and_preserve_missing() { + mut manager := new_test_manager(false) + placeholders := { + 'title': 'Hello' + 'body': '' + } + rendered := manager.expand('page.html', placeholders: &placeholders) + assert rendered == '
    Hello <script>alert(1)</script> @missing
    \n' + assert manager.compiled_template_count() == 1 +} + +fn test_include_and_compiled_template_registry() { + mut manager := new_test_manager(true) + placeholders := { + 'title': 'Home' + } + rendered := manager.expand('with_include.html', placeholders: &placeholders) + assert rendered == '
    Home
    ' + assert manager.compiled_template_count() == 1 + _ = manager.expand('with_include.html', placeholders: &placeholders) + assert manager.compiled_template_count() == 1 +} + +fn test_include_html_suffix_keeps_only_allowed_tags() { + mut manager := new_test_manager(false) + placeholders := { + 'body_#includehtml': 'safe' + 'title': 'HTML' + } + rendered := manager.expand('page.html', placeholders: &placeholders) + assert rendered.contains('safe') + assert rendered.contains('<script>bad</script>') +} + +fn test_text_mode_always_escapes_html() { + mut manager := new_test_manager(false) + placeholders := { + 'body_#includehtml': 'text' + } + rendered := manager.expand('raw.txt', placeholders: &placeholders) + assert rendered == 'Raw: <span>text</span>\n' +} + +fn test_reload_modified_template_without_render_cache() { + mut manager := new_test_manager(false) + placeholders := { + 'title': 'One' + } + first := manager.expand('reload.html', placeholders: &placeholders) + assert first == '

    One

    \n' + os.write_file(os.join_path(dtm2_test_root(), 'reload.html'), '@title')! + second := manager.expand('reload.html', placeholders: &placeholders) + assert second == 'One\n' + assert manager.compiled_template_count() == 1 +} + +fn test_reload_modified_include_without_render_cache() { + mut manager := new_test_manager(true) + placeholders := { + 'title': 'Home' + } + first := manager.expand('with_include.html', placeholders: &placeholders) + assert first == '
    Home
    ' + time.sleep(1100 * time.millisecond) + os.write_file(os.join_path(dtm2_test_root(), 'partials', 'nav.html'), + '')! + second := manager.expand('with_include.html', placeholders: &placeholders) + assert second == '
    Home
    ' + assert manager.compiled_template_count() == 1 +} + +fn test_reload_disabled_keeps_cached_template_tree() { + mut manager := initialize( + template_dir: dtm2_test_root() + compress_html: false + reload_modified_templates: false + ) + placeholders := { + 'title': 'Pinned' + } + first := manager.expand('pinned.html', placeholders: &placeholders) + assert first == 'Pinned\n' + os.write_file(os.join_path(dtm2_test_root(), 'pinned.html'), '@title')! + second := manager.expand('pinned.html', placeholders: &placeholders) + assert second == 'Pinned\n' + assert manager.compiled_template_count() == 1 +} + +fn test_relative_and_absolute_paths_share_compiled_template() { + mut manager := new_test_manager(false) + placeholders := { + 'title': 'Alias' + } + relative := manager.expand('path_alias.html', placeholders: &placeholders) + absolute := manager.expand(os.join_path(dtm2_test_root(), 'path_alias.html'), + placeholders: &placeholders + ) + assert relative == '
    Alias
    \n' + assert absolute == relative + assert manager.compiled_template_count() == 1 +} + +fn test_template_path_cannot_escape_template_dir() { + mut manager := new_test_manager(false) + placeholders := { + 'title': 'Escape' + } + relative_escape := manager.expand('../${os.base(dtm2_outside_template_path())}', + placeholders: &placeholders + ) + absolute_escape := manager.expand(dtm2_outside_template_path(), placeholders: &placeholders) + assert relative_escape == internal_server_error + assert absolute_escape == internal_server_error + assert manager.compiled_template_count() == 0 +} + +fn test_include_path_cannot_escape_template_dir() { + mut manager := new_test_manager(false) + placeholders := { + 'title': 'Escape' + } + relative_include := manager.expand('include_outside_relative.html', placeholders: &placeholders) + absolute_include := manager.expand('include_outside_absolute.html', placeholders: &placeholders) + assert relative_include == internal_server_error + assert absolute_include == internal_server_error + assert manager.compiled_template_count() == 0 +} + +fn test_custom_missing_placeholder_prefix() { + mut manager := new_test_manager(false) + placeholders := map[string]string{} + rendered := manager.expand('prefix.html', + placeholders: &placeholders + missing_placeholder_prefix: '?' + ) + assert rendered == '

    ?unknown

    \n' +} + +fn test_recursive_include_returns_internal_server_error() { + mut manager := new_test_manager(false) + placeholders := map[string]string{} + rendered := manager.expand('recursive.html', placeholders: &placeholders) + assert rendered == internal_server_error + assert manager.compiled_template_count() == 0 +} + +fn test_html_compression_is_deterministic() { + mut manager := new_test_manager(true) + placeholders := { + 'title': 'Compact' + } + rendered := manager.expand('compress.html', placeholders: &placeholders) + assert rendered == '
    Compact
    ' +} + +fn test_xml_template_uses_html_rendering_mode() { + mut manager := new_test_manager(false) + placeholders := { + 'title': '' + } + rendered := manager.expand('feed.xml', placeholders: &placeholders) + assert rendered == '<escaped>\n' +} + +fn test_custom_template_extension_map() { + mut manager := initialize( + template_dir: dtm2_test_root() + template_extensions: { + '.page': TemplateType.html + 'note': TemplateType.text + } + ) + html_placeholders := { + 'title_#includehtml': 'custom' + } + text_placeholders := { + 'body_#includehtml': 'custom' + } + html := manager.expand('custom.page', placeholders: &html_placeholders) + text := manager.expand('custom.note', placeholders: &text_placeholders) + assert html.contains('custom<script>blocked</script>') + assert text == 'Note: <span>custom</span>\n' +} + +fn test_json_extension_config_file() { + mut manager := initialize( + template_dir: dtm2_test_root() + extension_config_file: os.join_path(dtm2_test_root(), 'extensions.json') + ) + placeholders := { + 'title': '' + } + html := manager.expand('custom.view', placeholders: &placeholders) + text := manager.expand('custom.mail', placeholders: &placeholders) + assert html == '
    <Config>
    ' + assert text == 'Subject: <Config>\n' +} + +fn test_default_json_extension_config_file_is_loaded_from_template_dir() { + mut manager := initialize( + template_dir: dtm2_test_root() + compress_html: false + ) + placeholders := { + 'title': '' + } + html := manager.expand('auto.fragment', placeholders: &placeholders) + text := manager.expand('auto.message', placeholders: &placeholders) + assert html == '\n' + assert text == 'Auto: <Auto>\n' +} + +fn test_invalid_json_extension_entries_are_ignored() { + mut manager := initialize( + template_dir: dtm2_test_root() + extension_config_file: os.join_path(dtm2_test_root(), 'invalid_extensions.json') + compress_html: false + ) + placeholders := { + 'title': '' + } + ignored := manager.expand('bad.path', placeholders: &placeholders) + loaded := manager.expand('auto.message', placeholders: &placeholders) + assert ignored == internal_server_error + assert loaded == 'Auto: <Safe>\n' +} + +fn test_missing_template_returns_internal_server_error() { + mut manager := new_test_manager(false) + placeholders := map[string]string{} + rendered := manager.expand('missing.html', placeholders: &placeholders) + assert rendered == internal_server_error + assert manager.compiled_template_count() == 0 +} + +fn test_invalid_template_extension_returns_internal_server_error() { + mut manager := new_test_manager(false) + placeholders := map[string]string{} + rendered := manager.expand('invalid.css', placeholders: &placeholders) + assert rendered == internal_server_error + assert manager.compiled_template_count() == 0 +} From e661512d8671f45dbdabcc7faf733ecb75122dca Mon Sep 17 00:00:00 2001 From: GGRei Date: Tue, 28 Apr 2026 21:53:32 +0200 Subject: [PATCH 2/3] dtm2: fix template cache path and content validation --- .../dtm2/dynamic_template_manager2.v | 69 +++++++++++++------ .../dtm2/dynamic_template_manager2_test.v | 36 ++++++++++ 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/vlib/x/templating/dtm2/dynamic_template_manager2.v b/vlib/x/templating/dtm2/dynamic_template_manager2.v index fc76a37931ec41..429af70e14a821 100644 --- a/vlib/x/templating/dtm2/dynamic_template_manager2.v +++ b/vlib/x/templating/dtm2/dynamic_template_manager2.v @@ -1,7 +1,8 @@ module dtm2 -import os +import hash import json +import os import strings // dtm2 is the modern runtime renderer for Dynamic Template Manager. @@ -62,9 +63,10 @@ pub: // template is still fresh. Includes are tracked as dependencies of the parent // template, so editing a partial can invalidate the compiled parent tree. struct TemplateDependency { - path string - modified_at i64 - size u64 + path string + modified_at i64 + size u64 + content_hash u64 } // CompiledTemplate is the parsed representation stored by Manager. @@ -303,11 +305,18 @@ pub fn (mut m Manager) expand(template_path string, params RenderParams) string return rendered.clone() } -// cached_template_path resolves caller paths once and then reuses the canonical -// filesystem path. This removes repeated path normalization from the hot render -// loop while still keeping template content reload decisions separate. +// cached_template_path reuses canonical paths when reload checks are disabled. +// When reload checks are enabled, it revalidates the current real path so a +// replaced symlink cannot bypass the template directory boundary. fn (mut m Manager) cached_template_path(template_path string) !string { if source_path := m.resolved_template_paths[template_path] { + if m.reload_modified_templates { + current_source_path := m.resolve_template_path(template_path)! + if current_source_path != source_path { + m.resolved_template_paths[template_path] = current_source_path + return current_source_path + } + } return source_path } source_path := m.resolve_template_path(template_path)! @@ -320,7 +329,7 @@ fn (mut m Manager) cached_template_path(template_path string) !string { // tree can be reused or must be rebuilt from disk. fn (mut m Manager) compiled_template_for_path(source_path string) !&CompiledTemplate { if compiled := m.compiled_templates[source_path] { - if !m.reload_modified_templates || compiled.dependencies_are_fresh() { + if !m.reload_modified_templates || compiled.dependencies_are_fresh(m.template_dir) { return compiled } } @@ -332,8 +341,10 @@ fn (mut m Manager) compiled_template_for_path(source_path string) !&CompiledTemp // compile_template_from_file reads a template, expands static includes, parses // placeholders, and stores dependency metadata for future invalidation. fn compile_template_from_file(source_path string, template_root string, template_extensions map[string]TemplateType) !&CompiledTemplate { - template_type := template_type_from_path(source_path, template_extensions)! - raw_content, dependencies := read_template_with_includes(source_path, template_root, 0)! + canonical_path := os.real_path(source_path) + ensure_path_inside_template_dir(canonical_path, template_root)! + template_type := template_type_from_path(canonical_path, template_extensions)! + raw_content, dependencies := read_template_with_includes(canonical_path, template_root, 0)! instructions, estimated_size, segment_count := parse_segments(raw_content) dependency_signature := encode_dependencies(dependencies) return &CompiledTemplate{ @@ -358,24 +369,35 @@ fn copy_string(value string) string { return copied.clone() } -// dependencies_are_fresh checks root and included files. Both modification time -// and file size are used so same-second rewrites still invalidate reliably. -fn (compiled &CompiledTemplate) dependencies_are_fresh() bool { +// dependencies_are_fresh checks root and included files. The content hash closes +// the same-second, same-size edit window left by second-resolution mtimes. +fn (compiled &CompiledTemplate) dependencies_are_fresh(template_root string) bool { signature := compiled.dependency_signature mut offset := 0 for offset < signature.len { first_sep := index_byte_from(signature, `|`, offset) or { return false } second_sep := index_byte_from(signature, `|`, first_sep + 1) or { return false } - line_end := index_byte_from(signature, `\n`, second_sep + 1) or { signature.len } + third_sep := index_byte_from(signature, `|`, second_sep + 1) or { return false } + line_end := index_byte_from(signature, `\n`, third_sep + 1) or { signature.len } size := signature[first_sep + 1..second_sep].i64() if size < 0 { return false } - path := signature[second_sep + 1..line_end] + content_hash := signature[second_sep + 1..third_sep].u64() + path := signature[third_sep + 1..line_end] + current_path := os.real_path(path) + ensure_path_inside_template_dir(current_path, template_root) or { return false } + if current_path != path { + return false + } stat := os.stat(path) or { return false } if stat.mtime != signature[offset..first_sep].i64() || stat.size != u64(size) { return false } + content := os.read_file(path) or { return false } + if content_fingerprint(content) != content_hash { + return false + } offset = line_end + 1 } return true @@ -393,7 +415,7 @@ fn index_byte_from(value string, needle u8, start int) ?int { fn encode_dependencies(dependencies []TemplateDependency) string { mut out := strings.new_builder(dependencies.len * 64) for dependency in dependencies { - out.writeln('${dependency.modified_at}|${dependency.size}|${dependency.path}') + out.writeln('${dependency.modified_at}|${dependency.size}|${dependency.content_hash}|${dependency.path}') } encoded := out.str() return encoded.clone() @@ -451,7 +473,7 @@ fn read_template_with_includes(source_path string, template_root string, depth i base_dir := os.dir(source_path) mut out := strings.new_builder(content.len) mut dependencies := []TemplateDependency{cap: 4} - dependencies << template_dependency(source_path)! + dependencies << template_dependency(source_path, content)! lines := content.split_into_lines() for line in lines { expanded_line, line_dependencies := expand_include_directives(line, base_dir, @@ -464,15 +486,20 @@ fn read_template_with_includes(source_path string, template_root string, depth i return rendered.clone(), dependencies } -fn template_dependency(source_path string) !TemplateDependency { +fn template_dependency(source_path string, content string) !TemplateDependency { stat := os.stat(source_path)! return TemplateDependency{ - path: source_path.clone() - modified_at: stat.mtime - size: stat.size + path: source_path.clone() + modified_at: stat.mtime + size: stat.size + content_hash: content_fingerprint(content) } } +fn content_fingerprint(content string) u64 { + return hash.sum64_string(content, 0) +} + fn expand_include_directives(line string, base_dir string, template_root string, depth int) !(string, []TemplateDependency) { mut out := strings.new_builder(line.len) mut dependencies := []TemplateDependency{} diff --git a/vlib/x/templating/dtm2/dynamic_template_manager2_test.v b/vlib/x/templating/dtm2/dynamic_template_manager2_test.v index ee1ebbeb795b34..cc1f48db9c02ed 100644 --- a/vlib/x/templating/dtm2/dynamic_template_manager2_test.v +++ b/vlib/x/templating/dtm2/dynamic_template_manager2_test.v @@ -118,6 +118,24 @@ fn test_reload_modified_template_without_render_cache() { assert manager.compiled_template_count() == 1 } +fn test_reload_same_second_same_size_template_content_change() { + path := os.join_path(dtm2_test_root(), 'same_size.html') + fixed_time := i64(2_306_102_495) + os.write_file(path, '

    @title A

    ')! + os.utime(path, fixed_time, fixed_time)! + mut manager := new_test_manager(false) + placeholders := { + 'title': 'One' + } + first := manager.expand('same_size.html', placeholders: &placeholders) + assert first == '

    One A

    \n' + os.write_file(path, '@title B')! + os.utime(path, fixed_time, fixed_time)! + second := manager.expand('same_size.html', placeholders: &placeholders) + assert second == 'One B\n' + assert manager.compiled_template_count() == 1 +} + fn test_reload_modified_include_without_render_cache() { mut manager := new_test_manager(true) placeholders := { @@ -178,6 +196,24 @@ fn test_template_path_cannot_escape_template_dir() { assert manager.compiled_template_count() == 0 } +fn test_cached_template_path_revalidates_symlink_escape() { + $if windows { + return + } + path := os.join_path(dtm2_test_root(), 'symlink_swap.html') + os.write_file(path, '

    @title safe

    ')! + mut manager := new_test_manager(false) + placeholders := { + 'title': 'Link' + } + first := manager.expand('symlink_swap.html', placeholders: &placeholders) + assert first == '

    Link safe

    \n' + os.rm(path)! + os.symlink(dtm2_outside_template_path(), path) or { return } + escaped := manager.expand('symlink_swap.html', placeholders: &placeholders) + assert escaped == internal_server_error +} + fn test_include_path_cannot_escape_template_dir() { mut manager := new_test_manager(false) placeholders := { From 54fc8d0b8882c6409d8910df27132b706c2d5b12 Mon Sep 17 00:00:00 2001 From: GGRei Date: Tue, 28 Apr 2026 22:22:58 +0200 Subject: [PATCH 3/3] dtm2: optimize cold template loading and strengthen cache validation --- .../dtm2/dynamic_template_manager2.v | 102 +++++++++++++----- .../dtm2/dynamic_template_manager2_test.v | 87 +++++++++++++++ 2 files changed, 160 insertions(+), 29 deletions(-) diff --git a/vlib/x/templating/dtm2/dynamic_template_manager2.v b/vlib/x/templating/dtm2/dynamic_template_manager2.v index 429af70e14a821..714b9b3a88a47e 100644 --- a/vlib/x/templating/dtm2/dynamic_template_manager2.v +++ b/vlib/x/templating/dtm2/dynamic_template_manager2.v @@ -18,6 +18,7 @@ import strings const message_signature = '[Dynamic Template Manager 2]' const internal_server_error = 'Internal Server Error' const include_html_key_suffix = '_#includehtml' +const include_directive = '@include ' const max_include_depth = 32 const max_extension_config_size = 64 * 1024 const default_extension_config_filename = 'dtm2_extensions.json' @@ -109,8 +110,8 @@ mut: resolved_template_paths map[string]string // Parsed-template cache keyed by canonical source path. compiled_templates map[string]&CompiledTemplate - // Extension-to-render-mode table. Users can extend or override it through - // ManagerParams without changing the parser or renderer. + // User-provided extension-to-render-mode overrides. Built-in extensions are + // resolved without allocating a per-manager default map. template_extensions map[string]TemplateType } @@ -176,7 +177,7 @@ fn canonical_template_dir(template_dir string) string { } fn build_template_extensions(params ManagerParams, template_dir string) map[string]TemplateType { - mut extensions := default_template_extensions() + mut extensions := map[string]TemplateType{} config_path := extension_config_path(params, template_dir) if config_path != '' { file_extensions := read_extension_config_file(config_path) or { @@ -200,16 +201,6 @@ fn extension_config_path(params ManagerParams, template_dir string) string { return '' } -fn default_template_extensions() map[string]TemplateType { - return { - '.html': .html - '.htm': .html - '.xml': .html - '.txt': .text - '.text': .text - } -} - fn read_extension_config_file(config_path string) !map[string]TemplateType { if !os.exists(config_path) { return error('extension config file "${config_path}" not found') @@ -333,23 +324,25 @@ fn (mut m Manager) compiled_template_for_path(source_path string) !&CompiledTemp return compiled } } - compiled := compile_template_from_file(source_path, m.template_dir, m.template_extensions)! + compiled := compile_template_from_file(source_path, m.template_dir, m.template_extensions, + m.reload_modified_templates)! m.compiled_templates[source_path] = compiled return compiled } // compile_template_from_file reads a template, expands static includes, parses // placeholders, and stores dependency metadata for future invalidation. -fn compile_template_from_file(source_path string, template_root string, template_extensions map[string]TemplateType) !&CompiledTemplate { +fn compile_template_from_file(source_path string, template_root string, template_extensions map[string]TemplateType, track_dependencies bool) !&CompiledTemplate { canonical_path := os.real_path(source_path) ensure_path_inside_template_dir(canonical_path, template_root)! template_type := template_type_from_path(canonical_path, template_extensions)! - raw_content, dependencies := read_template_with_includes(canonical_path, template_root, 0)! - instructions, estimated_size, segment_count := parse_segments(raw_content) + content, dependencies := read_template_with_includes(canonical_path, template_root, 0, + track_dependencies)! + instructions, estimated_size, segment_count := parse_segments(content) dependency_signature := encode_dependencies(dependencies) return &CompiledTemplate{ template_type: template_type - content: copy_string(raw_content) + content: copy_string(content) dependency_signature: copy_string(dependency_signature) instructions: copy_string(instructions) segment_count: segment_count @@ -456,28 +449,52 @@ fn path_without_trailing_separator(path string) string { // template_type_from_path keeps rendering rules extension-driven and explicit. fn template_type_from_path(source_path string, template_extensions map[string]TemplateType) !TemplateType { ext := normalize_template_extension(os.file_ext(source_path)) - if template_type := template_extensions[ext] { + if template_extensions.len > 0 { + if template_type := template_extensions[ext] { + return template_type + } + } + if template_type := default_template_type_from_extension(ext) { return template_type } return error('template "${source_path}" uses unsupported extension "${ext}"') } +fn default_template_type_from_extension(ext string) ?TemplateType { + match ext { + '.html', '.htm', '.xml' { + return .html + } + '.txt', '.text' { + return .text + } + else { + return none + } + } +} + // read_template_with_includes expands simple line-level `@include "path"` // directives before parsing placeholders. Includes are part of the parsed tree // and are tracked as dependencies for reload checks. -fn read_template_with_includes(source_path string, template_root string, depth int) !(string, []TemplateDependency) { +fn read_template_with_includes(source_path string, template_root string, depth int, track_dependencies bool) !(string, []TemplateDependency) { if depth > max_include_depth { return error('maximum @include depth exceeded while reading "${source_path}"') } content := os.read_file(source_path)! + mut dependencies := []TemplateDependency{cap: 4} + if track_dependencies { + dependencies << template_dependency(source_path, content)! + } + if !content_needs_include_expansion(content) { + return content_with_template_newline(content), dependencies + } base_dir := os.dir(source_path) mut out := strings.new_builder(content.len) - mut dependencies := []TemplateDependency{cap: 4} - dependencies << template_dependency(source_path, content)! lines := content.split_into_lines() for line in lines { expanded_line, line_dependencies := expand_include_directives(line, base_dir, - template_root, depth)! + template_root, depth, track_dependencies)! out.write_string(expanded_line) out.write_u8(`\n`) dependencies << line_dependencies @@ -486,6 +503,33 @@ fn read_template_with_includes(source_path string, template_root string, depth i return rendered.clone(), dependencies } +fn content_with_template_newline(content string) string { + if content == '' || content[content.len - 1] == `\n` { + return content.clone() + } + with_newline := content + '\n' + return with_newline.clone() +} + +fn content_needs_include_expansion(content string) bool { + for i := 0; i < content.len; i++ { + if content[i] == `\r` { + return true + } + if is_include_directive_at(content, i) { + return true + } + } + return false +} + +fn is_include_directive_at(content string, pos int) bool { + return pos + include_directive.len <= content.len && content[pos] == `@` + && content[pos + 1] == `i` && content[pos + 2] == `n` && content[pos + 3] == `c` + && content[pos + 4] == `l` && content[pos + 5] == `u` && content[pos + 6] == `d` + && content[pos + 7] == `e` && content[pos + 8] == ` ` +} + fn template_dependency(source_path string, content string) !TemplateDependency { stat := os.stat(source_path)! return TemplateDependency{ @@ -500,26 +544,26 @@ fn content_fingerprint(content string) u64 { return hash.sum64_string(content, 0) } -fn expand_include_directives(line string, base_dir string, template_root string, depth int) !(string, []TemplateDependency) { +fn expand_include_directives(line string, base_dir string, template_root string, depth int, track_dependencies bool) !(string, []TemplateDependency) { mut out := strings.new_builder(line.len) mut dependencies := []TemplateDependency{} mut offset := 0 for { - rel_pos := line[offset..].index('@include ') or { + rel_pos := line[offset..].index(include_directive) or { out.write_string(line[offset..]) break } pos := offset + rel_pos target, end_pos := include_target_from_line_at(line, pos) or { - out.write_string(line[offset..pos + '@include '.len]) - offset = pos + '@include '.len + out.write_string(line[offset..pos + include_directive.len]) + offset = pos + include_directive.len continue } out.write_string(line[offset..pos]) include_path := resolve_include_path(base_dir, target, template_root)! next_depth := depth + 1 included, include_dependencies := read_template_with_includes(include_path, template_root, - next_depth)! + next_depth, track_dependencies)! out.write_string(included.trim_right('\n')) dependencies << include_dependencies offset = end_pos @@ -529,7 +573,7 @@ fn expand_include_directives(line string, base_dir string, template_root string, } fn include_target_from_line_at(line string, pos int) ?(string, int) { - mut cursor := pos + '@include '.len + mut cursor := pos + include_directive.len for cursor < line.len && line[cursor].is_space() { cursor++ } diff --git a/vlib/x/templating/dtm2/dynamic_template_manager2_test.v b/vlib/x/templating/dtm2/dynamic_template_manager2_test.v index cc1f48db9c02ed..6f5d5d342917c8 100644 --- a/vlib/x/templating/dtm2/dynamic_template_manager2_test.v +++ b/vlib/x/templating/dtm2/dynamic_template_manager2_test.v @@ -35,9 +35,11 @@ fn testsuite_begin() { os.write_file(os.join_path(root, 'prefix.html'), '

    @unknown

    ')! os.write_file(os.join_path(root, 'recursive.html'), '@include "recursive"')! os.write_file(os.join_path(root, 'compress.html'), '
    \n @title\n
    ')! + os.write_file(os.join_path(root, 'empty.html'), '')! os.write_file(os.join_path(root, 'feed.xml'), '@title')! os.write_file(os.join_path(root, 'custom.page'), '
    @title
    ')! os.write_file(os.join_path(root, 'custom.note'), 'Note: @body')! + os.write_file(os.join_path(root, 'override.html'), '
    \n @title\n
    ')! os.write_file(os.join_path(root, 'custom.view'), '
    @title
    ')! os.write_file(os.join_path(root, 'custom.mail'), 'Subject: @title')! os.write_file(os.join_path(root, 'auto.fragment'), '')! @@ -136,6 +138,26 @@ fn test_reload_same_second_same_size_template_content_change() { assert manager.compiled_template_count() == 1 } +fn test_reload_same_second_same_size_include_content_change() { + parent_path := os.join_path(dtm2_test_root(), 'same_size_include_parent.html') + include_path := os.join_path(dtm2_test_root(), 'partials', 'same_size_include.html') + fixed_time := i64(2_306_102_496) + os.write_file(parent_path, '
    @include "partials/same_size_include"
    ')! + os.write_file(include_path, '

    @title A

    ')! + os.utime(include_path, fixed_time, fixed_time)! + mut manager := new_test_manager(false) + placeholders := { + 'title': 'One' + } + first := manager.expand('same_size_include_parent.html', placeholders: &placeholders) + assert first == '

    One A

    \n' + os.write_file(include_path, '@title B')! + os.utime(include_path, fixed_time, fixed_time)! + second := manager.expand('same_size_include_parent.html', placeholders: &placeholders) + assert second == '
    One B
    \n' + assert manager.compiled_template_count() == 1 +} + fn test_reload_modified_include_without_render_cache() { mut manager := new_test_manager(true) placeholders := { @@ -214,6 +236,49 @@ fn test_cached_template_path_revalidates_symlink_escape() { assert escaped == internal_server_error } +fn test_cached_include_dependency_revalidates_symlink_escape() { + $if windows { + return + } + parent_path := os.join_path(dtm2_test_root(), 'include_symlink_swap_parent.html') + include_path := os.join_path(dtm2_test_root(), 'partials', 'symlink_swap.html') + os.write_file(parent_path, '
    @include "partials/symlink_swap"
    ')! + os.write_file(include_path, '

    @title safe

    ')! + mut manager := new_test_manager(false) + placeholders := { + 'title': 'Link' + } + first := manager.expand('include_symlink_swap_parent.html', placeholders: &placeholders) + assert first == '

    Link safe

    \n' + os.rm(include_path)! + os.symlink(dtm2_outside_template_path(), include_path) or { return } + escaped := manager.expand('include_symlink_swap_parent.html', placeholders: &placeholders) + assert escaped == internal_server_error +} + +fn test_reload_disabled_does_not_reopen_cached_symlink_swap() { + $if windows { + return + } + path := os.join_path(dtm2_test_root(), 'pinned_symlink_swap.html') + os.write_file(path, '

    @title safe

    ')! + mut manager := initialize( + template_dir: dtm2_test_root() + compress_html: false + reload_modified_templates: false + ) + placeholders := { + 'title': 'Pinned' + } + first := manager.expand('pinned_symlink_swap.html', placeholders: &placeholders) + assert first == '

    Pinned safe

    \n' + os.rm(path)! + os.symlink(dtm2_outside_template_path(), path) or { return } + second := manager.expand('pinned_symlink_swap.html', placeholders: &placeholders) + assert second == first + assert manager.compiled_template_count() == 1 +} + fn test_include_path_cannot_escape_template_dir() { mut manager := new_test_manager(false) placeholders := { @@ -253,6 +318,13 @@ fn test_html_compression_is_deterministic() { assert rendered == '
    Compact
    ' } +fn test_empty_template_renders_empty() { + mut manager := new_test_manager(false) + placeholders := map[string]string{} + rendered := manager.expand('empty.html', placeholders: &placeholders) + assert rendered == '' +} + fn test_xml_template_uses_html_rendering_mode() { mut manager := new_test_manager(false) placeholders := { @@ -282,6 +354,21 @@ fn test_custom_template_extension_map() { assert text == 'Note: <span>custom</span>\n' } +fn test_custom_template_extension_map_can_override_builtin_mapping() { + mut manager := initialize( + template_dir: dtm2_test_root() + compress_html: true + template_extensions: { + '.html': TemplateType.text + } + ) + placeholders := { + 'title': 'Text' + } + rendered := manager.expand('override.html', placeholders: &placeholders) + assert rendered == '
    \n Text\n
    \n' +} + fn test_json_extension_config_file() { mut manager := initialize( template_dir: dtm2_test_root()