DEVPROD-4926 add use_xlarge_distro option to mongo-task-generator (#71)

DEVPROD-4926 add use_xlarge_distro option to mongo-task-generator

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.7.11 - 2024-02-16
+* DEVPROD-4926 add use_xlarge_distro option to mongo-task-generator
+
 ## 0.7.10 - 2022-09-25
 * SERVER-81436 Read multiversion config from file instead of resmoke output.
 

Cargo.toml

@@ -2,7 +2,7 @@
 name = "mongo-task-generator"
 description = "Dynamically split evergreen tasks into subtasks for testing the mongodb/mongo project."
 license = "Apache-2.0"
-version = "0.7.10"
+version = "0.7.11"
 repository = "https://github.com/mongodb/mongo-task-generator"
 authors = ["Decision Automation Group <[email protected]>"]
 edition = "2018"

docs/generating_tasks.md

@@ -101,7 +101,7 @@ Looking at a [sample resmoke-based](https://github.com/mongodb/mongo/blob/852c5d
 ```
 
 Like fuzzer tasks, task generation is indicated by including the `"generate resmoke tasks"` function.
-Additionally, the 2 parameters here will impact how the task is generated.
+Additionally, the 3 parameters here will impact how the task is generated.
 
 * **suite**: By default, the name of the task (with the `_gen` suffix stripped off) will be used
   to determine which resmoke suite to base the generated sub-tasks on. This can be overwritten with
@@ -110,6 +110,10 @@ Additionally, the 2 parameters here will impact how the task is generated.
   successfully. When generated sub-tasks are run on build_variants with a `large_distro_name`
   expansion defined, they will run on that large distro instead of the default distro by setting
   the `use_large_distro` variable to `"true"`.
+* **use_xlarge_distro**: For when `use_large_distro` is not enough. When the `use_xlarge_distro`
+  variable is set to `"true"`, certain tasks will use an even larger distro that can be defined with
+  the `xlarge_distro_name` expansion in the build variant. When the `xlarge_distro_name` expansion
+  is not defined, it will fallback to the defined `large_distro_name` expansion in the build variant
 
 **Note**: If a task has the `use_large_distro` value defined, but is added to a build variant
 without a `large_distro_name`, it will trigger a failure. This can be supported by using the

src/evergreen_names.rs

@@ -50,6 +50,8 @@ pub const BURN_IN_TASKS: &str = "burn_in_tasks_gen";
 pub const IS_FUZZER: &str = "is_jstestfuzz";
 /// If true, generate sub-tasks to run on large distros.
 pub const USE_LARGE_DISTRO: &str = "use_large_distro";
+/// If true, generate sub-tasks to run on large distros.
+pub const USE_XLARGE_DISTRO: &str = "use_xlarge_distro";
 /// Number of files that each fuzzer sub-task should generate.
 pub const NUM_FUZZER_FILES: &str = "num_files";
 /// Number of sub-tasks that should be generated for a fuzzer.
@@ -93,6 +95,8 @@ pub const MULTIVERSION_EXCLUDE_TAGS: &str = "multiversion_exclude_tags_version";
 // Build Variant expansions.
 /// Name of large distro for build variant.
 pub const LARGE_DISTRO_EXPANSION: &str = "large_distro_name";
+/// Name of xlarge distro for build variant.
+pub const XLARGE_DISTRO_EXPANSION: &str = "xlarge_distro_name";
 /// List of build variant names delimited by spaces to generate burn_in_tests for.
 pub const BURN_IN_TAG_INCLUDE_BUILD_VARIANTS: &str = "burn_in_tag_include_build_variants";
 /// Generate burn_in_tests for all required and suggested build variants.

src/services/config_extraction.rs

@@ -9,6 +9,7 @@ use crate::{
         CONTINUE_ON_FAILURE, FUZZER_PARAMETERS, IDLE_TIMEOUT, LARGE_DISTRO_EXPANSION, MULTIVERSION,
         NO_MULTIVERSION_GENERATE_TASKS, NPM_COMMAND, NUM_FUZZER_FILES, NUM_FUZZER_TASKS,
         REPEAT_SUITES, RESMOKE_ARGS, RESMOKE_JOBS_MAX, SHOULD_SHUFFLE_TESTS, USE_LARGE_DISTRO,
+        USE_XLARGE_DISTRO, XLARGE_DISTRO_EXPANSION,
     },
     generate_sub_tasks_config::GenerateSubTasksConfig,
     task_types::{
@@ -235,6 +236,11 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl {
                 USE_LARGE_DISTRO,
                 false,
             )?,
+            use_xlarge_distro: self.evg_config_utils.lookup_default_param_bool(
+                task_def,
+                USE_XLARGE_DISTRO,
+                false,
+            )?,
             require_multiversion_setup,
             require_multiversion_generate_tasks: require_multiversion_setup
                 && !no_multiversion_generate_tasks,
@@ -286,9 +292,16 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl {
         let large_distro_name = self
             .evg_config_utils
             .lookup_build_variant_expansion(LARGE_DISTRO_EXPANSION, build_variant);
+        let xlarge_distro_name = self
+            .evg_config_utils
+            .lookup_build_variant_expansion(XLARGE_DISTRO_EXPANSION, build_variant);
         let build_variant_name = build_variant.name.as_str();
 
-        if generated_task.use_large_distro() {
+        if generated_task.use_xlarge_distro() && xlarge_distro_name.is_some() {
+            return Ok(xlarge_distro_name);
+        }
+
+        if generated_task.use_large_distro() || generated_task.use_xlarge_distro() {
             if large_distro_name.is_some() {
                 return Ok(large_distro_name);
             }
@@ -414,6 +427,7 @@ mod tests {
                         ..Default::default()
                     },
                     use_large_distro: *value,
+                    use_xlarge_distro: false,
                 })
                 .collect(),
         };
@@ -444,6 +458,7 @@ mod tests {
                     ..Default::default()
                 },
                 use_large_distro: true,
+                use_xlarge_distro: false,
             }],
         };
         let build_variant = BuildVariant {
@@ -474,6 +489,7 @@ mod tests {
                     ..Default::default()
                 },
                 use_large_distro: true,
+                use_xlarge_distro: false,
             }],
         };
         let build_variant = BuildVariant {

src/task_types/fuzzer_tasks.rs

@@ -168,6 +168,7 @@ impl GeneratedSuite for FuzzerTask {
             .map(|sub_task| GeneratedSubTask {
                 evg_task: sub_task,
                 use_large_distro: false,
+                use_xlarge_distro: false,
             })
             .collect()
     }

src/task_types/generated_suite.rs

@@ -10,6 +10,8 @@ pub struct GeneratedSubTask {
     pub evg_task: EvgTask,
     /// Whether to run generated task on a large distro.
     pub use_large_distro: bool,
+    /// Whether to run generated task on a xlarge distro.
+    pub use_xlarge_distro: bool,
 }
 
 /// Interface for representing a generated task.
@@ -27,6 +29,13 @@ pub trait GeneratedSuite: Sync + Send {
             .any(|sub_task| sub_task.use_large_distro)
     }
 
+    /// Check whether any sub task requires xlarge distro.
+    fn use_xlarge_distro(&self) -> bool {
+        self.sub_tasks()
+            .iter()
+            .any(|sub_task| sub_task.use_xlarge_distro)
+    }
+
     /// Build a shrub display task for this generated task.
     fn build_display_task(&self) -> DisplayTask {
         DisplayTask {
@@ -45,7 +54,7 @@ pub trait GeneratedSuite: Sync + Send {
             .iter()
             .map(|sub_task| {
                 let mut large_distro = None;
-                if sub_task.use_large_distro {
+                if sub_task.use_large_distro || sub_task.use_xlarge_distro {
                     large_distro = distro.clone();
                 }
                 sub_task

src/task_types/resmoke_tasks.rs

@@ -52,6 +52,8 @@ pub struct ResmokeGenParams {
     pub suite_name: String,
     /// Should the generated tasks run on a 'large' distro.
     pub use_large_distro: bool,
+    /// Should the generated tasks run on a 'xlarge' distro.
+    pub use_xlarge_distro: bool,
     /// Does this task require multiversion setup.
     pub require_multiversion_setup: bool,
     /// Should multiversion generate tasks exist for this.
@@ -776,6 +778,7 @@ impl GenResmokeTaskService for GenResmokeTaskServiceImpl {
                 ..Default::default()
             },
             use_large_distro: params.use_large_distro,
+            use_xlarge_distro: params.use_xlarge_distro,
         }
     }
 }
@@ -1014,6 +1017,7 @@ mod tests {
                         ..Default::default()
                     },
                     use_large_distro: *value,
+                    use_xlarge_distro: false,
                 })
                 .collect(),
         };