Nested SLURM Jobs#
executorlib extends the Executor interface from the Python standard library’s concurrent.futures to distribute Python functions as jobs on an HPC cluster. This notebook submits a single outer SLURM job via SlurmClusterExecutor, which itself opens a second, nested SlurmJobExecutor to fan out many short-lived tasks as job steps within that same allocation. See the HPC Job Executor documentation for details on nesting executors.
Based on the cmti and cmmg clusters hosted at the MPCDF for the MPI for Sustainable Materials.
import executorlib
from time import sleep
def add_with_sleep(i):
sleep(5)
return i + i
add_with_sleep is a toy workload: it sleeps for 5 seconds to simulate a short unit of computation, then returns i + i. Its purpose is to demonstrate scheduling overhead, not to compute anything meaningful.
def nested_executor(workers=10, rangemax=200):
with executorlib.SlurmJobExecutor(max_workers=workers) as exe:
future_lst = []
for i in range(rangemax):
future_lst.append(exe.submit(add_with_sleep, i))
return [f.result() for f in future_lst]
nested_executor runs inside the outer SLURM job and itself creates a SlurmJobExecutor, which dispatches tasks as srun job steps within the already-running allocation instead of submitting new sbatch jobs. With max_workers=10, up to ten add_with_sleep calls run concurrently, cycling through all rangemax inputs before the results are collected.
submission_template = """\
#!/bin/bash
#SBATCH --output=time.out
#SBATCH --job-name={{job_name}}
#SBATCH --chdir={{working_directory}}
#SBATCH --get-user-env
#SBATCH --partition={{partition}}
{%- if run_time_max %}
#SBATCH --time={{ [1, run_time_max // 60]|max }}
{%- endif %}
{%- if dependency %}
#SBATCH --dependency=afterok:{{ dependency | join(',') }}
{%- endif %}
{%- if memory_max %}
#SBATCH --mem={{memory_max}}G
{%- endif %}
#SBATCH --ntasks={{cores}}
{{command}}
"""
This is the standard Jinja2 sbatch submission template for the outer job. #SBATCH --ntasks={{cores}} maps executorlib’s cores resource onto the number of SLURM tasks requested; --mem, --time, and --dependency are only emitted when those keys are present in resource_dict.
with executorlib.SlurmClusterExecutor() as exe:
f1 = exe.submit(
nested_executor,
workers=10,
rangemax=200,
resource_dict={
"submission_template": submission_template,
"run_time_max": 180, # in seconds
# "partition": "s.cmfe",
"partition": "s.cmmg",
"cores": 1,
"threads_per_core": 10,
})
result = f1.result()
print(result)
/cmmc/ptmp/janj/mambaforge/envs/research/lib/python3.12/site-packages/executorlib/task_scheduler/base.py:156: UserWarning: The following keys are not recognized and cannot be validated: ['partition']
self._validator(resource_dict=resource_dict)
[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 160, 162, 164, 166, 168, 170, 172, 174, 176, 178, 180, 182, 184, 186, 188, 190, 192, 194, 196, 198, 200, 202, 204, 206, 208, 210, 212, 214, 216, 218, 220, 222, 224, 226, 228, 230, 232, 234, 236, 238, 240, 242, 244, 246, 248, 250, 252, 254, 256, 258, 260, 262, 264, 266, 268, 270, 272, 274, 276, 278, 280, 282, 284, 286, 288, 290, 292, 294, 296, 298, 300, 302, 304, 306, 308, 310, 312, 314, 316, 318, 320, 322, 324, 326, 328, 330, 332, 334, 336, 338, 340, 342, 344, 346, 348, 350, 352, 354, 356, 358, 360, 362, 364, 366, 368, 370, 372, 374, 376, 378, 380, 382, 384, 386, 388, 390, 392, 394, 396, 398]
The outer job is submitted with cores=1, threads_per_core=10, giving the allocation 10 CPU threads — enough for the nested SlurmJobExecutor’s max_workers=10 to each run one add_with_sleep step concurrently. nested_executor is called with workers=10, rangemax=200, so all 200 doubled values are returned as a single list once the outer job’s Future resolves.
executorlib.get_cache_data("executorlib_cache")
[{'function': <function __main__.nested_executor(workers=10, rangemax=200)>,
'input_args': [],
'input_kwargs': {'workers': 10, 'rangemax': 200},
'output': [0,
2,
4,
6,
8,
10,
12,
14,
16,
18,
20,
22,
24,
26,
28,
30,
32,
34,
36,
38,
40,
42,
44,
46,
48,
50,
52,
54,
56,
58,
60,
62,
64,
66,
68,
70,
72,
74,
76,
78,
80,
82,
84,
86,
88,
90,
92,
94,
96,
98,
100,
102,
104,
106,
108,
110,
112,
114,
116,
118,
120,
122,
124,
126,
128,
130,
132,
134,
136,
138,
140,
142,
144,
146,
148,
150,
152,
154,
156,
158,
160,
162,
164,
166,
168,
170,
172,
174,
176,
178,
180,
182,
184,
186,
188,
190,
192,
194,
196,
198,
200,
202,
204,
206,
208,
210,
212,
214,
216,
218,
220,
222,
224,
226,
228,
230,
232,
234,
236,
238,
240,
242,
244,
246,
248,
250,
252,
254,
256,
258,
260,
262,
264,
266,
268,
270,
272,
274,
276,
278,
280,
282,
284,
286,
288,
290,
292,
294,
296,
298,
300,
302,
304,
306,
308,
310,
312,
314,
316,
318,
320,
322,
324,
326,
328,
330,
332,
334,
336,
338,
340,
342,
344,
346,
348,
350,
352,
354,
356,
358,
360,
362,
364,
366,
368,
370,
372,
374,
376,
378,
380,
382,
384,
386,
388,
390,
392,
394,
396,
398],
'resource_dict': {'submission_template': "#!/bin/bash\n#SBATCH --output=time.out\n#SBATCH --job-name={{job_name}}\n#SBATCH --chdir={{working_directory}}\n#SBATCH --get-user-env\n#SBATCH --partition={{partition}}\n{%- if run_time_max %}\n#SBATCH --time={{ [1, run_time_max // 60]|max }}\n{%- endif %}\n{%- if dependency %}\n#SBATCH --dependency=afterok:{{ dependency | join(',') }}\n{%- endif %}\n{%- if memory_max %}\n#SBATCH --mem={{memory_max}}G\n{%- endif %}\n#SBATCH --ntasks={{cores}}\n\n{{command}}\n",
'run_time_max': 180,
'partition': 's.cmmg',
'cores': 1,
'threads_per_core': 10,
'gpus_per_core': 0,
'cwd': None,
'openmpi_oversubscribe': False,
'slurm_cmd_args': [],
'exclusive': False},
'runtime': 116.25651049613953,
'queue_id': 21567759,
'error_log_file': None,
'filename': '/cmmc/u/janj/notebooks/2026/2026-07-28-executor-slurm/executorlib_cache/nested_executor676922b03c3d8501e29cf9abd928f8c6_o.h5'}]
get_cache_data reads the on-disk cache executorlib writes for submitted functions. Only the outer nested_executor call appears here, together with its resource_dict, wall-clock runtime, and SLURM queue_id — the 200 inner add_with_sleep steps run inside the same allocation via SlurmJobExecutor and are not tracked as separate cache entries. See the caching section of the installation guide for more on how the cache is configured.