Skip to content
Snippets Groups Projects

Patch causing hang in MPI

  • Clone with SSH
  • Clone with HTTPS
  • Embed
  • Share
    The snippet can be accessed without any authentication.
    Authored by Matt Wala

    Run with

    py.test --capture=no test_distributed.py

    Result:

    ==================================================================== test session starts =====================================================================
platform linux -- Python 3.6.5, pytest-3.6.2, py-1.5.4, pluggy-0.6.0
rootdir: /home/wala1/src/boxtree, inifile:
collected 2 items                                                                                                                                            

test_distributed.py Cannot open file '/home/wala1/src/boxtree/boxtree/distributed/default_perf_model.json'
Traceback (most recent call last):
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/site-packages/mpi4py/run.py", line 206, in <module>
    main()
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/site-packages/mpi4py/run.py", line 196, in main
    run_command_line(args)
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/site-packages/mpi4py/run.py", line 47, in run_command_line
    run_path(sys.argv[0], run_name='__main__')
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/runpy.py", line 263, in run_path
    pkg_name=pkg_name, script_name=fname)
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/home/wala1/miniconda3/envs/inteq/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/wala1/src/boxtree/test/test_distributed.py", line 371, in <module>
    _test_against_shared(dims, nsources, ntargets, dtype)
  File "/home/wala1/src/boxtree/test/test_distributed.py", line 80, in _test_against_shared
    queue, trav, distributed_expansion_wrangler_factory, comm=comm)
  File "/home/wala1/src/boxtree/boxtree/distributed/__init__.py", line 106, in __init__
    boxes_time = model.predict_boxes_time(global_trav, self.global_wrangler)
  File "/home/wala1/src/boxtree/boxtree/distributed/perf_model.py", line 610, in predict_boxes_time
    param = self.eval_direct_model()
  File "/home/wala1/src/boxtree/boxtree/distributed/perf_model.py", line 438, in eval_direct_model
    wall_time=wall_time)
  File "/home/wala1/src/boxtree/boxtree/distributed/perf_model.py", line 473, in linear_regression
    raise RuntimeError("Please run FMM at lease once using time_performance"
RuntimeError: Please run FMM at lease once using time_performancebefore forming models.
--------------------------------------------------------------------------
MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD
with errorcode 1.

NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.
You may or may not see output from other processes, depending on
exactly when Open MPI kills them.
--------------------------------------------------------------------------
    hang.patch 55.70 KiB
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21 
    diff --git a/boxtree/distributed/default_perf_model.json b/boxtree/distributed/default_perf_model.json
deleted file mode 100644
index 5c6a449..0000000
--- a/boxtree/distributed/default_perf_model.json
+++ /dev/null
@@ -1 +0,0 @@
-[{"nterms_fmm_total": 1000000, "direct_workload": 5064570, "direct_nsource_boxes": 56907, "m2l_workload": 879645000, "m2p_workload": 80054000, "m2p_nboxes": 148660, "p2l_workload": 58602800, "p2l_nboxes": 84384, "eval_part_workload": 1000000, "form_multipoles": {"wall_elapsed": 0.14289305033162236, "process_elapsed": 0.09281096999999994}, "coarsen_multipoles": {"wall_elapsed": 0.14258357882499695, "process_elapsed": 0.14258194200000007}, "eval_direct": {"wall_elapsed": 2.344248099718243, "process_elapsed": 2.378837522999998}, "multipole_to_local": {"wall_elapsed": 20.023932092823088, "process_elapsed": 19.927228304}, "eval_multipoles": {"wall_elapsed": 3.287798510864377, "process_elapsed": 3.2817736969999984}, "form_locals": {"wall_elapsed": 2.141686537768692, "process_elapsed": 2.1379926530000013}, "refine_locals": {"wall_elapsed": 0.10454159695655107, "process_elapsed": 0.10454057600000155}, "eval_locals": {"wall_elapsed": 0.05774546507745981, "process_elapsed": 0.05774528099999898}}, {"nterms_fmm_total": 2000000, "direct_workload": 14111527, "direct_nsource_boxes": 105337, "m2l_workload": 1702853000, "m2p_workload": 202584200, "m2p_nboxes": 311728, "p2l_workload": 120805600, "p2l_nboxes": 146698, "eval_part_workload": 2000000, "form_multipoles": {"wall_elapsed": 0.10452838707715273, "process_elapsed": 0.10452839599999919}, "coarsen_multipoles": {"wall_elapsed": 0.21106432611122727, "process_elapsed": 0.21548113200000074}, "eval_direct": {"wall_elapsed": 4.7187186549417675, "process_elapsed": 4.702423059000012}, "multipole_to_local": {"wall_elapsed": 40.50634287390858, "process_elapsed": 40.512730489999996}, "eval_multipoles": {"wall_elapsed": 9.251218600198627, "process_elapsed": 9.216514626000006}, "form_locals": {"wall_elapsed": 4.5346991759724915, "process_elapsed": 4.524306796000005}, "refine_locals": {"wall_elapsed": 0.1888848263770342, "process_elapsed": 0.19806307300000014}, "eval_locals": {"wall_elapsed": 0.10716611426323652, "process_elapsed": 0.10716471899999647}}, {"nterms_fmm_total": 3000000, "direct_workload": 24428758, "direct_nsource_boxes": 223172, "m2l_workload": 4011675000, "m2p_workload": 670713600, "m2p_nboxes": 735176, "p2l_workload": 213069300, "p2l_nboxes": 259667, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.1347822081297636, "process_elapsed": 0.13477609299999926}, "coarsen_multipoles": {"wall_elapsed": 0.3603002396412194, "process_elapsed": 0.3602957989999993}, "eval_direct": {"wall_elapsed": 11.283871918916702, "process_elapsed": 11.367921355000007}, "multipole_to_local": {"wall_elapsed": 92.28659388935193, "process_elapsed": 92.02833609099999}, "eval_multipoles": {"wall_elapsed": 24.550387303810567, "process_elapsed": 24.45371944300001}, "form_locals": {"wall_elapsed": 7.136191665194929, "process_elapsed": 7.125686079000019}, "refine_locals": {"wall_elapsed": 0.42898024804890156, "process_elapsed": 0.42143030599999065}, "eval_locals": {"wall_elapsed": 0.27421190217137337, "process_elapsed": 0.2742086969999775}}, {"nterms_fmm_total": 4000000, "direct_workload": 29106936, "direct_nsource_boxes": 259688, "m2l_workload": 4679517000, "m2p_workload": 737745100, "m2p_nboxes": 887508, "p2l_workload": 262131700, "p2l_nboxes": 277715, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.2584445010870695, "process_elapsed": 0.25844339400001104}, "coarsen_multipoles": {"wall_elapsed": 0.578245738055557, "process_elapsed": 0.5796970419999923}, "eval_direct": {"wall_elapsed": 13.274465977214277, "process_elapsed": 13.333639903999995}, "multipole_to_local": {"wall_elapsed": 111.34071257105097, "process_elapsed": 110.98862347400001}, "eval_multipoles": {"wall_elapsed": 27.020350001286715, "process_elapsed": 27.001812247000032}, "form_locals": {"wall_elapsed": 9.117257341276854, "process_elapsed": 9.09303535500004}, "refine_locals": {"wall_elapsed": 0.4617841048166156, "process_elapsed": 0.45366404100002455}, "eval_locals": {"wall_elapsed": 0.29467571387067437, "process_elapsed": 0.29467265900001394}}, {"nterms_fmm_total": 5000000, "direct_workload": 47923958, "direct_nsource_boxes": 321217, "m2l_workload": 6098447000, "m2p_workload": 1049959000, "m2p_nboxes": 1058414, "p2l_workload": 250873800, "p2l_nboxes": 307399, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.3379204752855003, "process_elapsed": 0.3395408740000221}, "coarsen_multipoles": {"wall_elapsed": 0.7178796431981027, "process_elapsed": 0.7149586010000348}, "eval_direct": {"wall_elapsed": 15.559152766596526, "process_elapsed": 15.673222753999994}, "multipole_to_local": {"wall_elapsed": 140.75012073572725, "process_elapsed": 140.313214335}, "eval_multipoles": {"wall_elapsed": 36.27695396123454, "process_elapsed": 36.167522549999944}, "form_locals": {"wall_elapsed": 9.149377660825849, "process_elapsed": 9.114768321999918}, "refine_locals": {"wall_elapsed": 0.48122364515438676, "process_elapsed": 0.481217474999994}, "eval_locals": {"wall_elapsed": 0.3034700150601566, "process_elapsed": 0.3034582699999646}}, {"nterms_fmm_total": 6000000, "direct_workload": 52175740, "direct_nsource_boxes": 365978, "m2l_workload": 6963407000, "m2p_workload": 1228104100, "m2p_nboxes": 1238954, "p2l_workload": 320901400, "p2l_nboxes": 340870, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.2821324490942061, "process_elapsed": 0.2821300870000414}, "coarsen_multipoles": {"wall_elapsed": 0.7399735772050917, "process_elapsed": 0.7435758610000676}, "eval_direct": {"wall_elapsed": 17.014956682454795, "process_elapsed": 17.00256681099995}, "multipole_to_local": {"wall_elapsed": 159.9870855966583, "process_elapsed": 159.52702093099992}, "eval_multipoles": {"wall_elapsed": 44.760278538800776, "process_elapsed": 44.609669542999995}, "form_locals": {"wall_elapsed": 10.070859387051314, "process_elapsed": 10.042829486999949}, "refine_locals": {"wall_elapsed": 0.6633619191125035, "process_elapsed": 0.6633519270000079}, "eval_locals": {"wall_elapsed": 0.32876442186534405, "process_elapsed": 0.3287510919999477}}, {"nterms_fmm_total": 7000000, "direct_workload": 67754792, "direct_nsource_boxes": 399866, "m2l_workload": 7699296000, "m2p_workload": 1423135200, "m2p_nboxes": 1352554, "p2l_workload": 330677000, "p2l_nboxes": 356695, "eval_part_workload": 7000000, "form_multipoles": {"wall_elapsed": 0.34098224015906453, "process_elapsed": 0.3368965309999794}, "coarsen_multipoles": {"wall_elapsed": 0.8644498628564179, "process_elapsed": 0.8597816169999533}, "eval_direct": {"wall_elapsed": 20.44144478905946, "process_elapsed": 20.493337145999817}, "multipole_to_local": {"wall_elapsed": 176.60086810868233, "process_elapsed": 175.99911542799998}, "eval_multipoles": {"wall_elapsed": 47.3556498978287, "process_elapsed": 47.21309775099985}, "form_locals": {"wall_elapsed": 12.22528616571799, "process_elapsed": 12.173807162000003}, "refine_locals": {"wall_elapsed": 0.7411672458983958, "process_elapsed": 0.7411488990001089}, "eval_locals": {"wall_elapsed": 0.4227954070083797, "process_elapsed": 0.42279098400013027}}, {"nterms_fmm_total": 8000000, "direct_workload": 82707417, "direct_nsource_boxes": 441394, "m2l_workload": 8617704000, "m2p_workload": 1685913200, "m2p_nboxes": 1498566, "p2l_workload": 348565200, "p2l_nboxes": 382521, "eval_part_workload": 8000000, "form_multipoles": {"wall_elapsed": 0.43783256597816944, "process_elapsed": 0.4303805439999451}, "coarsen_multipoles": {"wall_elapsed": 0.9156720908358693, "process_elapsed": 0.9222488749999229}, "eval_direct": {"wall_elapsed": 21.642432290129364, "process_elapsed": 21.649963951000245}, "multipole_to_local": {"wall_elapsed": 200.9743533632718, "process_elapsed": 200.288029092}, "eval_multipoles": {"wall_elapsed": 54.97431806195527, "process_elapsed": 54.881239913999934}, "form_locals": {"wall_elapsed": 10.682431893888861, "process_elapsed": 10.645913471000085}, "refine_locals": {"wall_elapsed": 0.7400978719815612, "process_elapsed": 0.7279401089999737}, "eval_locals": {"wall_elapsed": 0.47856780607253313, "process_elapsed": 0.4785622940000849}}, {"nterms_fmm_total": 1000000, "direct_workload": 5064570, "direct_nsource_boxes": 56907, "m2l_workload": 879645000, "m2p_workload": 80054000, "m2p_nboxes": 148660, "p2l_workload": 58602800, "p2l_nboxes": 84384, "eval_part_workload": 1000000, "form_multipoles": {"wall_elapsed": 0.06467467220500112, "process_elapsed": 0.061806205999999975}, "coarsen_multipoles": {"wall_elapsed": 0.1378657571040094, "process_elapsed": 0.13786446399999974}, "eval_direct": {"wall_elapsed": 2.628749551717192, "process_elapsed": 2.6928994460000024}, "multipole_to_local": {"wall_elapsed": 20.407855125609785, "process_elapsed": 20.420269581}, "eval_multipoles": {"wall_elapsed": 3.4444818547926843, "process_elapsed": 3.4479039949999972}, "form_locals": {"wall_elapsed": 2.021404864266515, "process_elapsed": 2.013269193000003}, "refine_locals": {"wall_elapsed": 0.11385017307475209, "process_elapsed": 0.11385017100000283}, "eval_locals": {"wall_elapsed": 0.061653067357838154, "process_elapsed": 0.06165442400000032}}, {"nterms_fmm_total": 2000000, "direct_workload": 14111527, "direct_nsource_boxes": 105337, "m2l_workload": 1702853000, "m2p_workload": 202584200, "m2p_nboxes": 311728, "p2l_workload": 120805600, "p2l_nboxes": 146698, "eval_part_workload": 2000000, "form_multipoles": {"wall_elapsed": 0.10336571699008346, "process_elapsed": 0.10382350900000148}, "coarsen_multipoles": {"wall_elapsed": 0.18445956613868475, "process_elapsed": 0.18445694999999773}, "eval_direct": {"wall_elapsed": 5.217078930698335, "process_elapsed": 5.1945506220000155}, "multipole_to_local": {"wall_elapsed": 40.47428226983175, "process_elapsed": 40.43935276}, "eval_multipoles": {"wall_elapsed": 8.246962024830282, "process_elapsed": 8.253471535999992}, "form_locals": {"wall_elapsed": 4.244218919891864, "process_elapsed": 4.237957393000002}, "refine_locals": {"wall_elapsed": 0.20841451222077012, "process_elapsed": 0.20840436699999998}, "eval_locals": {"wall_elapsed": 0.12309996783733368, "process_elapsed": 0.12310028500000669}}, {"nterms_fmm_total": 3000000, "direct_workload": 24428758, "direct_nsource_boxes": 223172, "m2l_workload": 4011675000, "m2p_workload": 670713600, "m2p_nboxes": 735176, "p2l_workload": 213069300, "p2l_nboxes": 259667, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.24232071777805686, "process_elapsed": 0.24232055199999536}, "coarsen_multipoles": {"wall_elapsed": 0.5347794652916491, "process_elapsed": 0.534767346999999}, "eval_direct": {"wall_elapsed": 11.002750042360276, "process_elapsed": 11.097017833000024}, "multipole_to_local": {"wall_elapsed": 93.03417005809024, "process_elapsed": 92.738502075}, "eval_multipoles": {"wall_elapsed": 23.674448810052127, "process_elapsed": 23.61579378899998}, "form_locals": {"wall_elapsed": 7.417238333728164, "process_elapsed": 7.402527147000001}, "refine_locals": {"wall_elapsed": 0.3950768308714032, "process_elapsed": 0.39506440099998485}, "eval_locals": {"wall_elapsed": 0.25366233196109533, "process_elapsed": 0.25365634300001716}}, {"nterms_fmm_total": 4000000, "direct_workload": 29106936, "direct_nsource_boxes": 259688, "m2l_workload": 4679517000, "m2p_workload": 737745100, "m2p_nboxes": 887508, "p2l_workload": 262131700, "p2l_nboxes": 277715, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.17537363106384873, "process_elapsed": 0.17537425200001167}, "coarsen_multipoles": {"wall_elapsed": 0.4305199389345944, "process_elapsed": 0.4364680989999954}, "eval_direct": {"wall_elapsed": 11.999836526811123, "process_elapsed": 12.09899144499991}, "multipole_to_local": {"wall_elapsed": 108.8945693182759, "process_elapsed": 108.54051004800002}, "eval_multipoles": {"wall_elapsed": 28.0945353070274, "process_elapsed": 28.019863450000003}, "form_locals": {"wall_elapsed": 8.115772506222129, "process_elapsed": 8.086782484999958}, "refine_locals": {"wall_elapsed": 0.4050322100520134, "process_elapsed": 0.4050208720000228}, "eval_locals": {"wall_elapsed": 0.23690649028867483, "process_elapsed": 0.23690572800001064}}, {"nterms_fmm_total": 5000000, "direct_workload": 47923958, "direct_nsource_boxes": 321217, "m2l_workload": 6098447000, "m2p_workload": 1049959000, "m2p_nboxes": 1058414, "p2l_workload": 250873800, "p2l_nboxes": 307399, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.2068675379268825, "process_elapsed": 0.20685269899996683}, "coarsen_multipoles": {"wall_elapsed": 0.6696573188528419, "process_elapsed": 0.664978437000002}, "eval_direct": {"wall_elapsed": 15.180561240762472, "process_elapsed": 15.243839906999938}, "multipole_to_local": {"wall_elapsed": 143.82901267288253, "process_elapsed": 143.340336542}, "eval_multipoles": {"wall_elapsed": 34.135265816003084, "process_elapsed": 34.04566944399994}, "form_locals": {"wall_elapsed": 8.900728145148605, "process_elapsed": 8.893034466000017}, "refine_locals": {"wall_elapsed": 0.5461942246183753, "process_elapsed": 0.546178929000007}, "eval_locals": {"wall_elapsed": 0.3527874890714884, "process_elapsed": 0.35278468699993937}}, {"nterms_fmm_total": 6000000, "direct_workload": 52175740, "direct_nsource_boxes": 365978, "m2l_workload": 6963407000, "m2p_workload": 1228104100, "m2p_nboxes": 1238954, "p2l_workload": 320901400, "p2l_nboxes": 340870, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.310679045971483, "process_elapsed": 0.3128285689999757}, "coarsen_multipoles": {"wall_elapsed": 0.6655385498888791, "process_elapsed": 0.6780369559999144}, "eval_direct": {"wall_elapsed": 17.763539463747293, "process_elapsed": 17.785153179000076}, "multipole_to_local": {"wall_elapsed": 165.45908340485767, "process_elapsed": 164.94912508800007}, "eval_multipoles": {"wall_elapsed": 41.00005980208516, "process_elapsed": 40.86370828999998}, "form_locals": {"wall_elapsed": 10.63184105977416, "process_elapsed": 10.620832092}, "refine_locals": {"wall_elapsed": 0.6163057168014348, "process_elapsed": 0.6162964320000128}, "eval_locals": {"wall_elapsed": 0.3811777690425515, "process_elapsed": 0.3816459009999562}}, {"nterms_fmm_total": 7000000, "direct_workload": 67754792, "direct_nsource_boxes": 399866, "m2l_workload": 7699296000, "m2p_workload": 1423135200, "m2p_nboxes": 1352554, "p2l_workload": 330677000, "p2l_nboxes": 356695, "eval_part_workload": 7000000, "form_multipoles": {"wall_elapsed": 0.2698205062188208, "process_elapsed": 0.2698129970000309}, "coarsen_multipoles": {"wall_elapsed": 0.6649344856850803, "process_elapsed": 0.6794030730000031}, "eval_direct": {"wall_elapsed": 18.069350454490632, "process_elapsed": 18.11652952300028}, "multipole_to_local": {"wall_elapsed": 179.39501194981858, "process_elapsed": 178.77733126399994}, "eval_multipoles": {"wall_elapsed": 49.19099767273292, "process_elapsed": 49.06879406899998}, "form_locals": {"wall_elapsed": 11.126400337088853, "process_elapsed": 11.094821377000017}, "refine_locals": {"wall_elapsed": 0.8388090138323605, "process_elapsed": 0.8387958670000444}, "eval_locals": {"wall_elapsed": 0.43714700592681766, "process_elapsed": 0.4371343509999406}}, {"nterms_fmm_total": 8000000, "direct_workload": 82707417, "direct_nsource_boxes": 441394, "m2l_workload": 8617704000, "m2p_workload": 1685913200, "m2p_nboxes": 1498566, "p2l_workload": 348565200, "p2l_nboxes": 382521, "eval_part_workload": 8000000, "form_multipoles": {"wall_elapsed": 0.3005179218016565, "process_elapsed": 0.3107753879999109}, "coarsen_multipoles": {"wall_elapsed": 0.7997653689235449, "process_elapsed": 0.8049937840000894}, "eval_direct": {"wall_elapsed": 20.99420260032639, "process_elapsed": 21.002320735999774}, "multipole_to_local": {"wall_elapsed": 201.31278445525095, "process_elapsed": 200.60458022299986}, "eval_multipoles": {"wall_elapsed": 56.057990666944534, "process_elapsed": 55.91957687800004}, "form_locals": {"wall_elapsed": 11.969875158276409, "process_elapsed": 11.920894822000037}, "refine_locals": {"wall_elapsed": 0.7045927383005619, "process_elapsed": 0.7050175569997919}, "eval_locals": {"wall_elapsed": 0.43921483773738146, "process_elapsed": 0.4392006999999012}}, {"nterms_fmm_total": 1000000, "direct_workload": 5064570, "direct_nsource_boxes": 56907, "m2l_workload": 879645000, "m2p_workload": 80054000, "m2p_nboxes": 148660, "p2l_workload": 58602800, "p2l_nboxes": 84384, "eval_part_workload": 1000000, "form_multipoles": {"wall_elapsed": 0.06566942017525434, "process_elapsed": 0.06264390200000003}, "coarsen_multipoles": {"wall_elapsed": 0.1418711910955608, "process_elapsed": 0.1418699069999998}, "eval_direct": {"wall_elapsed": 2.446558577939868, "process_elapsed": 2.480314300999999}, "multipole_to_local": {"wall_elapsed": 20.00079088192433, "process_elapsed": 19.990914497}, "eval_multipoles": {"wall_elapsed": 3.289981202688068, "process_elapsed": 3.298983691}, "form_locals": {"wall_elapsed": 1.949442199897021, "process_elapsed": 1.9376855149999983}, "refine_locals": {"wall_elapsed": 0.10793576203286648, "process_elapsed": 0.1079347100000021}, "eval_locals": {"wall_elapsed": 0.056184975896030664, "process_elapsed": 0.056184503999997304}}, {"nterms_fmm_total": 2000000, "direct_workload": 14111527, "direct_nsource_boxes": 105337, "m2l_workload": 1702853000, "m2p_workload": 202584200, "m2p_nboxes": 311728, "p2l_workload": 120805600, "p2l_nboxes": 146698, "eval_part_workload": 2000000, "form_multipoles": {"wall_elapsed": 0.0772943552583456, "process_elapsed": 0.07729414800000001}, "coarsen_multipoles": {"wall_elapsed": 0.1868803328834474, "process_elapsed": 0.18687748400000004}, "eval_direct": {"wall_elapsed": 4.70476703485474, "process_elapsed": 4.733044761000009}, "multipole_to_local": {"wall_elapsed": 39.82684602914378, "process_elapsed": 39.775730969}, "eval_multipoles": {"wall_elapsed": 8.290043313987553, "process_elapsed": 8.295265927000003}, "form_locals": {"wall_elapsed": 4.078609869815409, "process_elapsed": 4.073348116000005}, "refine_locals": {"wall_elapsed": 0.20782660599797964, "process_elapsed": 0.20782425500000556}, "eval_locals": {"wall_elapsed": 0.15716208703815937, "process_elapsed": 0.15309306700000036}}, {"nterms_fmm_total": 3000000, "direct_workload": 24428758, "direct_nsource_boxes": 223172, "m2l_workload": 4011675000, "m2p_workload": 670713600, "m2p_nboxes": 735176, "p2l_workload": 213069300, "p2l_nboxes": 259667, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.16780595015734434, "process_elapsed": 0.16779495699999813}, "coarsen_multipoles": {"wall_elapsed": 0.42514409590512514, "process_elapsed": 0.4251383390000001}, "eval_direct": {"wall_elapsed": 11.506238477304578, "process_elapsed": 11.525195629999985}, "multipole_to_local": {"wall_elapsed": 93.78711060807109, "process_elapsed": 93.47971988}, "eval_multipoles": {"wall_elapsed": 24.25627316115424, "process_elapsed": 24.202846137000023}, "form_locals": {"wall_elapsed": 7.886253128293902, "process_elapsed": 7.8745142149999765}, "refine_locals": {"wall_elapsed": 0.5174721670337021, "process_elapsed": 0.5133926010000209}, "eval_locals": {"wall_elapsed": 0.2859889171086252, "process_elapsed": 0.2859800949999851}}, {"nterms_fmm_total": 4000000, "direct_workload": 29106936, "direct_nsource_boxes": 259688, "m2l_workload": 4679517000, "m2p_workload": 737745100, "m2p_nboxes": 887508, "p2l_workload": 262131700, "p2l_nboxes": 277715, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.183073571883142, "process_elapsed": 0.18305776899998705}, "coarsen_multipoles": {"wall_elapsed": 0.472671234048903, "process_elapsed": 0.47271501700001295}, "eval_direct": {"wall_elapsed": 12.604105783626437, "process_elapsed": 12.64961513900002}, "multipole_to_local": {"wall_elapsed": 107.64262523688376, "process_elapsed": 107.31761293199997}, "eval_multipoles": {"wall_elapsed": 27.184093620162457, "process_elapsed": 27.179954468999995}, "form_locals": {"wall_elapsed": 8.9608427840285, "process_elapsed": 8.936076344000014}, "refine_locals": {"wall_elapsed": 0.559703144710511, "process_elapsed": 0.559695051999995}, "eval_locals": {"wall_elapsed": 0.25458914041519165, "process_elapsed": 0.2545796289999771}}, {"nterms_fmm_total": 5000000, "direct_workload": 47923958, "direct_nsource_boxes": 321217, "m2l_workload": 6098447000, "m2p_workload": 1049959000, "m2p_nboxes": 1058414, "p2l_workload": 250873800, "p2l_nboxes": 307399, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.2608643942512572, "process_elapsed": 0.262650323999992}, "coarsen_multipoles": {"wall_elapsed": 0.5767403277568519, "process_elapsed": 0.5860576580000156}, "eval_direct": {"wall_elapsed": 15.735880189575255, "process_elapsed": 15.813270451999927}, "multipole_to_local": {"wall_elapsed": 140.27258323831484, "process_elapsed": 139.77256453799998}, "eval_multipoles": {"wall_elapsed": 36.051918339915574, "process_elapsed": 35.95092203699994}, "form_locals": {"wall_elapsed": 8.086318483110517, "process_elapsed": 8.063137184000084}, "refine_locals": {"wall_elapsed": 0.6116273296065629, "process_elapsed": 0.6116195170000083}, "eval_locals": {"wall_elapsed": 0.30272550601512194, "process_elapsed": 0.30271572899994226}}, {"nterms_fmm_total": 6000000, "direct_workload": 52175740, "direct_nsource_boxes": 365978, "m2l_workload": 6963407000, "m2p_workload": 1228104100, "m2p_nboxes": 1238954, "p2l_workload": 320901400, "p2l_nboxes": 340870, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.23625991912558675, "process_elapsed": 0.2364244679999956}, "coarsen_multipoles": {"wall_elapsed": 0.6893667639233172, "process_elapsed": 0.6830092370000784}, "eval_direct": {"wall_elapsed": 15.056561090052128, "process_elapsed": 15.098359518000052}, "multipole_to_local": {"wall_elapsed": 161.72943416889757, "process_elapsed": 161.19251369200003}, "eval_multipoles": {"wall_elapsed": 41.702806482091546, "process_elapsed": 41.572584398000004}, "form_locals": {"wall_elapsed": 10.078389388974756, "process_elapsed": 10.070601187999955}, "refine_locals": {"wall_elapsed": 0.5847061406821012, "process_elapsed": 0.5846636710000439}, "eval_locals": {"wall_elapsed": 0.32613639906048775, "process_elapsed": 0.3261274469999762}}, {"nterms_fmm_total": 7000000, "direct_workload": 67754792, "direct_nsource_boxes": 399866, "m2l_workload": 7699296000, "m2p_workload": 1423135200, "m2p_nboxes": 1352554, "p2l_workload": 330677000, "p2l_nboxes": 356695, "eval_part_workload": 7000000, "form_multipoles": {"wall_elapsed": 0.35347751434892416, "process_elapsed": 0.3534746269999687}, "coarsen_multipoles": {"wall_elapsed": 0.7678062850609422, "process_elapsed": 0.7621317609999778}, "eval_direct": {"wall_elapsed": 19.385647067334503, "process_elapsed": 19.373809754000035}, "multipole_to_local": {"wall_elapsed": 177.7568790889345, "process_elapsed": 177.13995553400014}, "eval_multipoles": {"wall_elapsed": 55.13101848727092, "process_elapsed": 54.92711723799994}, "form_locals": {"wall_elapsed": 10.444787265732884, "process_elapsed": 10.436945378000019}, "refine_locals": {"wall_elapsed": 0.6166465007700026, "process_elapsed": 0.6166181649998634}, "eval_locals": {"wall_elapsed": 0.35686514899134636, "process_elapsed": 0.35686196599999676}}, {"nterms_fmm_total": 8000000, "direct_workload": 82707417, "direct_nsource_boxes": 441394, "m2l_workload": 8617704000, "m2p_workload": 1685913200, "m2p_nboxes": 1498566, "p2l_workload": 348565200, "p2l_nboxes": 382521, "eval_part_workload": 8000000, "form_multipoles": {"wall_elapsed": 0.3867368856444955, "process_elapsed": 0.3867698809999638}, "coarsen_multipoles": {"wall_elapsed": 0.7339309039525688, "process_elapsed": 0.7566592790001323}, "eval_direct": {"wall_elapsed": 20.32549550011754, "process_elapsed": 20.388905034000118}, "multipole_to_local": {"wall_elapsed": 198.88563758181408, "process_elapsed": 198.28836379799986}, "eval_multipoles": {"wall_elapsed": 51.132025649771094, "process_elapsed": 50.934892610999896}, "form_locals": {"wall_elapsed": 11.990623429883271, "process_elapsed": 11.949524165999946}, "refine_locals": {"wall_elapsed": 0.6882827966473997, "process_elapsed": 0.680187923999938}, "eval_locals": {"wall_elapsed": 0.40719516295939684, "process_elapsed": 0.40764913800012437}}, {"nterms_fmm_total": 1000000, "direct_workload": 5064570, "direct_nsource_boxes": 56907, "m2l_workload": 879645000, "m2p_workload": 80054000, "m2p_nboxes": 148660, "p2l_workload": 58602800, "p2l_nboxes": 84384, "eval_part_workload": 1000000, "form_multipoles": {"wall_elapsed": 0.06581627298146486, "process_elapsed": 0.06344213200000004}, "coarsen_multipoles": {"wall_elapsed": 0.1412572581321001, "process_elapsed": 0.1412559019999997}, "eval_direct": {"wall_elapsed": 2.7299221428111196, "process_elapsed": 2.781603758000003}, "multipole_to_local": {"wall_elapsed": 20.612939092796296, "process_elapsed": 20.616477141}, "eval_multipoles": {"wall_elapsed": 3.5307789859361947, "process_elapsed": 3.517346646}, "form_locals": {"wall_elapsed": 2.124925720039755, "process_elapsed": 2.1208257669999995}, "refine_locals": {"wall_elapsed": 0.1124834748916328, "process_elapsed": 0.11248237000000216}, "eval_locals": {"wall_elapsed": 0.06405873689800501, "process_elapsed": 0.0640584180000019}}, {"nterms_fmm_total": 2000000, "direct_workload": 14111527, "direct_nsource_boxes": 105337, "m2l_workload": 1702853000, "m2p_workload": 202584200, "m2p_nboxes": 311728, "p2l_workload": 120805600, "p2l_nboxes": 146698, "eval_part_workload": 2000000, "form_multipoles": {"wall_elapsed": 0.1036986019462347, "process_elapsed": 0.10369902800000119}, "coarsen_multipoles": {"wall_elapsed": 0.1949111670255661, "process_elapsed": 0.1999170879999994}, "eval_direct": {"wall_elapsed": 4.608586409594864, "process_elapsed": 4.5927578570000165}, "multipole_to_local": {"wall_elapsed": 38.54366623284295, "process_elapsed": 38.485827138000005}, "eval_multipoles": {"wall_elapsed": 8.568737780675292, "process_elapsed": 8.542159236999993}, "form_locals": {"wall_elapsed": 4.073263252153993, "process_elapsed": 4.064695062000013}, "refine_locals": {"wall_elapsed": 0.309948590118438, "process_elapsed": 0.3115177579999937}, "eval_locals": {"wall_elapsed": 0.1761215110309422, "process_elapsed": 0.17611938100000657}}, {"nterms_fmm_total": 3000000, "direct_workload": 24428758, "direct_nsource_boxes": 223172, "m2l_workload": 4011675000, "m2p_workload": 670713600, "m2p_nboxes": 735176, "p2l_workload": 213069300, "p2l_nboxes": 259667, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.13820320600643754, "process_elapsed": 0.138196674999989}, "coarsen_multipoles": {"wall_elapsed": 0.38043196592479944, "process_elapsed": 0.3804273089999981}, "eval_direct": {"wall_elapsed": 11.744406177196652, "process_elapsed": 11.827659204999975}, "multipole_to_local": {"wall_elapsed": 91.26099151792005, "process_elapsed": 90.957502208}, "eval_multipoles": {"wall_elapsed": 22.827171033713967, "process_elapsed": 22.78696679800001}, "form_locals": {"wall_elapsed": 7.519823815207928, "process_elapsed": 7.498412040000005}, "refine_locals": {"wall_elapsed": 0.36121500795707107, "process_elapsed": 0.36120376799999576}, "eval_locals": {"wall_elapsed": 0.2246690890751779, "process_elapsed": 0.2246666040000207}}, {"nterms_fmm_total": 4000000, "direct_workload": 29106936, "direct_nsource_boxes": 259688, "m2l_workload": 4679517000, "m2p_workload": 737745100, "m2p_nboxes": 887508, "p2l_workload": 262131700, "p2l_nboxes": 277715, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.17177576525136828, "process_elapsed": 0.1717763049999803}, "coarsen_multipoles": {"wall_elapsed": 0.42073291214182973, "process_elapsed": 0.4207269609999855}, "eval_direct": {"wall_elapsed": 11.8608891190961, "process_elapsed": 11.953956640000058}, "multipole_to_local": {"wall_elapsed": 108.9518269430846, "process_elapsed": 108.638402584}, "eval_multipoles": {"wall_elapsed": 29.55046471906826, "process_elapsed": 29.447812095000018}, "form_locals": {"wall_elapsed": 8.121391328983009, "process_elapsed": 8.098203552999962}, "refine_locals": {"wall_elapsed": 0.40015376918017864, "process_elapsed": 0.40014009999998734}, "eval_locals": {"wall_elapsed": 0.24294947879388928, "process_elapsed": 0.24293507200002296}}, {"nterms_fmm_total": 5000000, "direct_workload": 47923958, "direct_nsource_boxes": 321217, "m2l_workload": 6098447000, "m2p_workload": 1049959000, "m2p_nboxes": 1058414, "p2l_workload": 250873800, "p2l_nboxes": 307399, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.3036691783927381, "process_elapsed": 0.3037372880000362}, "coarsen_multipoles": {"wall_elapsed": 0.7293630791828036, "process_elapsed": 0.7386667420000208}, "eval_direct": {"wall_elapsed": 15.853490174748003, "process_elapsed": 15.951854819999937}, "multipole_to_local": {"wall_elapsed": 150.07324582897127, "process_elapsed": 149.59256680499993}, "eval_multipoles": {"wall_elapsed": 36.21569347428158, "process_elapsed": 36.108943527000065}, "form_locals": {"wall_elapsed": 9.256117098033428, "process_elapsed": 9.234842302999937}, "refine_locals": {"wall_elapsed": 0.6879124888218939, "process_elapsed": 0.6842866189999768}, "eval_locals": {"wall_elapsed": 0.4285805160179734, "process_elapsed": 0.4285680850000517}}, {"nterms_fmm_total": 6000000, "direct_workload": 52175740, "direct_nsource_boxes": 365978, "m2l_workload": 6963407000, "m2p_workload": 1228104100, "m2p_nboxes": 1238954, "p2l_workload": 320901400, "p2l_nboxes": 340870, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.2390671377070248, "process_elapsed": 0.23906642099996134}, "coarsen_multipoles": {"wall_elapsed": 0.6618608850985765, "process_elapsed": 0.6946877950000498}, "eval_direct": {"wall_elapsed": 16.631962614599615, "process_elapsed": 16.672639716999925}, "multipole_to_local": {"wall_elapsed": 161.71785036334768, "process_elapsed": 161.19572901000004}, "eval_multipoles": {"wall_elapsed": 42.610538037959486, "process_elapsed": 42.503210351999996}, "form_locals": {"wall_elapsed": 11.49972936604172, "process_elapsed": 11.47165019199997}, "refine_locals": {"wall_elapsed": 0.7118247579783201, "process_elapsed": 0.7118074400000296}, "eval_locals": {"wall_elapsed": 0.380106495693326, "process_elapsed": 0.38009324099994046}}, {"nterms_fmm_total": 7000000, "direct_workload": 67754792, "direct_nsource_boxes": 399866, "m2l_workload": 7699296000, "m2p_workload": 1423135200, "m2p_nboxes": 1352554, "p2l_workload": 330677000, "p2l_nboxes": 356695, "eval_part_workload": 7000000, "form_multipoles": {"wall_elapsed": 0.3670531171374023, "process_elapsed": 0.36309775300003366}, "coarsen_multipoles": {"wall_elapsed": 0.7518898211419582, "process_elapsed": 0.7729549489999954}, "eval_direct": {"wall_elapsed": 18.767360630445182, "process_elapsed": 18.769919828999946}, "multipole_to_local": {"wall_elapsed": 180.2374455779791, "process_elapsed": 179.64930603300002}, "eval_multipoles": {"wall_elapsed": 49.90873555187136, "process_elapsed": 49.73379236400001}, "form_locals": {"wall_elapsed": 10.527730371803045, "process_elapsed": 10.505953223000006}, "refine_locals": {"wall_elapsed": 0.6305891978554428, "process_elapsed": 0.6305585160000646}, "eval_locals": {"wall_elapsed": 0.5451591932214797, "process_elapsed": 0.5386910560000615}}, {"nterms_fmm_total": 8000000, "direct_workload": 82707417, "direct_nsource_boxes": 441394, "m2l_workload": 8617704000, "m2p_workload": 1685913200, "m2p_nboxes": 1498566, "p2l_workload": 348565200, "p2l_nboxes": 382521, "eval_part_workload": 8000000, "form_multipoles": {"wall_elapsed": 0.3035329198464751, "process_elapsed": 0.3046291329999349}, "coarsen_multipoles": {"wall_elapsed": 0.908652248326689, "process_elapsed": 0.9229546799999753}, "eval_direct": {"wall_elapsed": 23.435453578364104, "process_elapsed": 23.427185823999707}, "multipole_to_local": {"wall_elapsed": 204.10029060393572, "process_elapsed": 203.498182929}, "eval_multipoles": {"wall_elapsed": 57.77209374681115, "process_elapsed": 57.580601317999935}, "form_locals": {"wall_elapsed": 11.694038683082908, "process_elapsed": 11.65377241200008}, "refine_locals": {"wall_elapsed": 0.6659307437948883, "process_elapsed": 0.6659000030001607}, "eval_locals": {"wall_elapsed": 0.42689173109829426, "process_elapsed": 0.4268864119999307}}, {"nterms_fmm_total": 1000000, "direct_workload": 5064570, "direct_nsource_boxes": 56907, "m2l_workload": 879645000, "m2p_workload": 80054000, "m2p_nboxes": 148660, "p2l_workload": 58602800, "p2l_nboxes": 84384, "eval_part_workload": 1000000, "form_multipoles": {"wall_elapsed": 0.06433802330866456, "process_elapsed": 0.06175603100000027}, "coarsen_multipoles": {"wall_elapsed": 0.13728277198970318, "process_elapsed": 0.1372830300000003}, "eval_direct": {"wall_elapsed": 2.8698996007442474, "process_elapsed": 2.8860750380000035}, "multipole_to_local": {"wall_elapsed": 21.608792692888528, "process_elapsed": 21.600729749}, "eval_multipoles": {"wall_elapsed": 4.170610999688506, "process_elapsed": 4.196137698000001}, "form_locals": {"wall_elapsed": 1.8854569140821695, "process_elapsed": 1.8855776410000011}, "refine_locals": {"wall_elapsed": 0.16010719677433372, "process_elapsed": 0.16010556499999495}, "eval_locals": {"wall_elapsed": 0.08334489725530148, "process_elapsed": 0.08334504000000464}}, {"nterms_fmm_total": 2000000, "direct_workload": 14111527, "direct_nsource_boxes": 105337, "m2l_workload": 1702853000, "m2p_workload": 202584200, "m2p_nboxes": 311728, "p2l_workload": 120805600, "p2l_nboxes": 146698, "eval_part_workload": 2000000, "form_multipoles": {"wall_elapsed": 0.07694227108731866, "process_elapsed": 0.07694427899999567}, "coarsen_multipoles": {"wall_elapsed": 0.18193004885688424, "process_elapsed": 0.18192722100000225}, "eval_direct": {"wall_elapsed": 5.362686685286462, "process_elapsed": 5.376335398999991}, "multipole_to_local": {"wall_elapsed": 38.82882813597098, "process_elapsed": 38.800395750999996}, "eval_multipoles": {"wall_elapsed": 7.6316725057549775, "process_elapsed": 7.61176749900001}, "form_locals": {"wall_elapsed": 3.7538683358579874, "process_elapsed": 3.740067400000001}, "refine_locals": {"wall_elapsed": 0.18044804502278566, "process_elapsed": 0.18044670099999394}, "eval_locals": {"wall_elapsed": 0.1640238557010889, "process_elapsed": 0.15998751500001163}}, {"nterms_fmm_total": 3000000, "direct_workload": 24428758, "direct_nsource_boxes": 223172, "m2l_workload": 4011675000, "m2p_workload": 670713600, "m2p_nboxes": 735176, "p2l_workload": 213069300, "p2l_nboxes": 259667, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.16299484204500914, "process_elapsed": 0.1589325700000046}, "coarsen_multipoles": {"wall_elapsed": 0.5394922369159758, "process_elapsed": 0.5394833269999992}, "eval_direct": {"wall_elapsed": 11.529483899474144, "process_elapsed": 11.633024222999964}, "multipole_to_local": {"wall_elapsed": 90.45034577790648, "process_elapsed": 90.140623747}, "eval_multipoles": {"wall_elapsed": 23.475032337009907, "process_elapsed": 23.402588277000007}, "form_locals": {"wall_elapsed": 7.382363630924374, "process_elapsed": 7.379083809000008}, "refine_locals": {"wall_elapsed": 0.3571728630922735, "process_elapsed": 0.3571698819999938}, "eval_locals": {"wall_elapsed": 0.29383321665227413, "process_elapsed": 0.28573974700000804}}, {"nterms_fmm_total": 4000000, "direct_workload": 29106936, "direct_nsource_boxes": 259688, "m2l_workload": 4679517000, "m2p_workload": 737745100, "m2p_nboxes": 887508, "p2l_workload": 262131700, "p2l_nboxes": 277715, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.17482035793364048, "process_elapsed": 0.17482164000000466}, "coarsen_multipoles": {"wall_elapsed": 0.6471779476851225, "process_elapsed": 0.6350906819999977}, "eval_direct": {"wall_elapsed": 11.94170841993764, "process_elapsed": 12.134813735999984}, "multipole_to_local": {"wall_elapsed": 109.43196559883654, "process_elapsed": 109.081074277}, "eval_multipoles": {"wall_elapsed": 29.13186590280384, "process_elapsed": 29.05222061400002}, "form_locals": {"wall_elapsed": 9.20513498224318, "process_elapsed": 9.17659013399998}, "refine_locals": {"wall_elapsed": 0.39085082802921534, "process_elapsed": 0.3908466930000145}, "eval_locals": {"wall_elapsed": 0.23075266415253282, "process_elapsed": 0.23075040099996613}}, {"nterms_fmm_total": 5000000, "direct_workload": 47923958, "direct_nsource_boxes": 321217, "m2l_workload": 6098447000, "m2p_workload": 1049959000, "m2p_nboxes": 1058414, "p2l_workload": 250873800, "p2l_nboxes": 307399, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.31254801992326975, "process_elapsed": 0.30992854599998054}, "coarsen_multipoles": {"wall_elapsed": 0.637142900377512, "process_elapsed": 0.6467619349999723}, "eval_direct": {"wall_elapsed": 15.69611760089174, "process_elapsed": 15.757981944999813}, "multipole_to_local": {"wall_elapsed": 140.356332520023, "process_elapsed": 139.88694511699998}, "eval_multipoles": {"wall_elapsed": 38.49987019971013, "process_elapsed": 38.43145631599998}, "form_locals": {"wall_elapsed": 8.471607562154531, "process_elapsed": 8.449020899000061}, "refine_locals": {"wall_elapsed": 0.5592385237105191, "process_elapsed": 0.5596502360000386}, "eval_locals": {"wall_elapsed": 0.35891397623345256, "process_elapsed": 0.3589064400000552}}, {"nterms_fmm_total": 6000000, "direct_workload": 52175740, "direct_nsource_boxes": 365978, "m2l_workload": 6963407000, "m2p_workload": 1228104100, "m2p_nboxes": 1238954, "p2l_workload": 320901400, "p2l_nboxes": 340870, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.22916135005652905, "process_elapsed": 0.2294245719999708}, "coarsen_multipoles": {"wall_elapsed": 0.5582886077463627, "process_elapsed": 0.5605387639999435}, "eval_direct": {"wall_elapsed": 16.712323531042784, "process_elapsed": 16.77518112300004}, "multipole_to_local": {"wall_elapsed": 159.60533997509629, "process_elapsed": 159.07230811499994}, "eval_multipoles": {"wall_elapsed": 42.59306370886043, "process_elapsed": 42.45572857700006}, "form_locals": {"wall_elapsed": 10.805573990568519, "process_elapsed": 10.770887148999918}, "refine_locals": {"wall_elapsed": 0.5874539171345532, "process_elapsed": 0.579410558999939}, "eval_locals": {"wall_elapsed": 0.34672214509919286, "process_elapsed": 0.3467195629999651}}, {"nterms_fmm_total": 7000000, "direct_workload": 67754792, "direct_nsource_boxes": 399866, "m2l_workload": 7699296000, "m2p_workload": 1423135200, "m2p_nboxes": 1352554, "p2l_workload": 330677000, "p2l_nboxes": 356695, "eval_part_workload": 7000000, "form_multipoles": {"wall_elapsed": 0.34223695332184434, "process_elapsed": 0.3381626640000377}, "coarsen_multipoles": {"wall_elapsed": 0.8546734559349716, "process_elapsed": 0.8620984199999384}, "eval_direct": {"wall_elapsed": 20.248594620265067, "process_elapsed": 20.2735790050001}, "multipole_to_local": {"wall_elapsed": 177.53715593600646, "process_elapsed": 176.916876098}, "eval_multipoles": {"wall_elapsed": 46.6420071949251, "process_elapsed": 46.50159333199986}, "form_locals": {"wall_elapsed": 11.556011468637735, "process_elapsed": 11.530447846000015}, "refine_locals": {"wall_elapsed": 0.6371138021349907, "process_elapsed": 0.6370984749999025}, "eval_locals": {"wall_elapsed": 0.48767920210957527, "process_elapsed": 0.4836109719999513}}, {"nterms_fmm_total": 8000000, "direct_workload": 82707417, "direct_nsource_boxes": 441394, "m2l_workload": 8617704000, "m2p_workload": 1685913200, "m2p_nboxes": 1498566, "p2l_workload": 348565200, "p2l_nboxes": 382521, "eval_part_workload": 8000000, "form_multipoles": {"wall_elapsed": 0.4101475323550403, "process_elapsed": 0.4025623400000313}, "coarsen_multipoles": {"wall_elapsed": 0.8309031310491264, "process_elapsed": 0.8533334829999148}, "eval_direct": {"wall_elapsed": 21.83228043373674, "process_elapsed": 21.811839603999942}, "multipole_to_local": {"wall_elapsed": 200.4263506392017, "process_elapsed": 199.78048179799998}, "eval_multipoles": {"wall_elapsed": 55.83001929195598, "process_elapsed": 55.62621917499996}, "form_locals": {"wall_elapsed": 10.937592420261353, "process_elapsed": 10.904243739000094}, "refine_locals": {"wall_elapsed": 0.8196568163111806, "process_elapsed": 0.8155691360000219}, "eval_locals": {"wall_elapsed": 0.6638513482175767, "process_elapsed": 0.6638208470001246}}, {"nterms_fmm_total": 1000000, "direct_workload": 5064570, "direct_nsource_boxes": 56907, "m2l_workload": 879645000, "m2p_workload": 80054000, "m2p_nboxes": 148660, "p2l_workload": 58602800, "p2l_nboxes": 84384, "eval_part_workload": 1000000, "form_multipoles": {"wall_elapsed": 0.06488013127818704, "process_elapsed": 0.06267741799999982}, "coarsen_multipoles": {"wall_elapsed": 0.14028360787779093, "process_elapsed": 0.1402826039999998}, "eval_direct": {"wall_elapsed": 3.573185256216675, "process_elapsed": 3.561920087999996}, "multipole_to_local": {"wall_elapsed": 25.873784495983273, "process_elapsed": 25.732638186}, "eval_multipoles": {"wall_elapsed": 4.642745970282704, "process_elapsed": 4.639341426000001}, "form_locals": {"wall_elapsed": 3.0461708339862525, "process_elapsed": 3.036363762999997}, "refine_locals": {"wall_elapsed": 0.11961846519261599, "process_elapsed": 0.1196096529999977}, "eval_locals": {"wall_elapsed": 0.0641988911665976, "process_elapsed": 0.06419945400000415}}, {"nterms_fmm_total": 2000000, "direct_workload": 14111527, "direct_nsource_boxes": 105337, "m2l_workload": 1702853000, "m2p_workload": 202584200, "m2p_nboxes": 311728, "p2l_workload": 120805600, "p2l_nboxes": 146698, "eval_part_workload": 2000000, "form_multipoles": {"wall_elapsed": 0.09974845498800278, "process_elapsed": 0.09974945999999818}, "coarsen_multipoles": {"wall_elapsed": 0.2711884528398514, "process_elapsed": 0.2711792780000053}, "eval_direct": {"wall_elapsed": 6.678008021786809, "process_elapsed": 6.685549811999991}, "multipole_to_local": {"wall_elapsed": 52.48578274901956, "process_elapsed": 50.94725895999999}, "eval_multipoles": {"wall_elapsed": 11.199469511862844, "process_elapsed": 10.973406965999999}, "form_locals": {"wall_elapsed": 5.428788446821272, "process_elapsed": 5.372336655000012}, "refine_locals": {"wall_elapsed": 0.2391572780907154, "process_elapsed": 0.2391561879999955}, "eval_locals": {"wall_elapsed": 0.13382326532155275, "process_elapsed": 0.13382537700000796}}, {"nterms_fmm_total": 3000000, "direct_workload": 24428758, "direct_nsource_boxes": 223172, "m2l_workload": 4011675000, "m2p_workload": 670713600, "m2p_nboxes": 735176, "p2l_workload": 213069300, "p2l_nboxes": 259667, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.17349608801305294, "process_elapsed": 0.17349610799999482}, "coarsen_multipoles": {"wall_elapsed": 0.4378826292231679, "process_elapsed": 0.43415366199999994}, "eval_direct": {"wall_elapsed": 14.007537546101958, "process_elapsed": 14.066462461}, "multipole_to_local": {"wall_elapsed": 131.48507186723873, "process_elapsed": 128.31762228600002}, "eval_multipoles": {"wall_elapsed": 29.94184113200754, "process_elapsed": 29.82207009000001}, "form_locals": {"wall_elapsed": 9.938785715959966, "process_elapsed": 9.913687585000048}, "refine_locals": {"wall_elapsed": 0.4504069094546139, "process_elapsed": 0.4487136799999689}, "eval_locals": {"wall_elapsed": 0.30359381902962923, "process_elapsed": 0.30358403499997166}}, {"nterms_fmm_total": 4000000, "direct_workload": 29106936, "direct_nsource_boxes": 259688, "m2l_workload": 4679517000, "m2p_workload": 737745100, "m2p_nboxes": 887508, "p2l_workload": 262131700, "p2l_nboxes": 277715, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.2583726462908089, "process_elapsed": 0.2584110270000224}, "coarsen_multipoles": {"wall_elapsed": 0.5752558228559792, "process_elapsed": 0.5769190489999687}, "eval_direct": {"wall_elapsed": 12.875543330796063, "process_elapsed": 12.883404596000048}, "multipole_to_local": {"wall_elapsed": 117.18043434806168, "process_elapsed": 116.57996186900004}, "eval_multipoles": {"wall_elapsed": 27.881489561870694, "process_elapsed": 27.78543085000001}, "form_locals": {"wall_elapsed": 8.799156446009874, "process_elapsed": 8.768620798000029}, "refine_locals": {"wall_elapsed": 0.4524097847752273, "process_elapsed": 0.45239475799996853}, "eval_locals": {"wall_elapsed": 0.2726343311369419, "process_elapsed": 0.27261175699999285}}, {"nterms_fmm_total": 5000000, "direct_workload": 47923958, "direct_nsource_boxes": 321217, "m2l_workload": 6098447000, "m2p_workload": 1049959000, "m2p_nboxes": 1058414, "p2l_workload": 250873800, "p2l_nboxes": 307399, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.22424806095659733, "process_elapsed": 0.2242476690000217}, "coarsen_multipoles": {"wall_elapsed": 0.6595294354483485, "process_elapsed": 0.6785909230000016}, "eval_direct": {"wall_elapsed": 16.094195554498583, "process_elapsed": 16.12589607299998}, "multipole_to_local": {"wall_elapsed": 144.88413401972502, "process_elapsed": 144.42795690799994}, "eval_multipoles": {"wall_elapsed": 37.05021429667249, "process_elapsed": 36.94682276699996}, "form_locals": {"wall_elapsed": 8.93320427602157, "process_elapsed": 8.92260437199991}, "refine_locals": {"wall_elapsed": 0.5139365317299962, "process_elapsed": 0.5144198460000098}, "eval_locals": {"wall_elapsed": 0.4673224021680653, "process_elapsed": 0.4632590369999434}}, {"nterms_fmm_total": 6000000, "direct_workload": 52175740, "direct_nsource_boxes": 365978, "m2l_workload": 6963407000, "m2p_workload": 1228104100, "m2p_nboxes": 1238954, "p2l_workload": 320901400, "p2l_nboxes": 340870, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.39084181608632207, "process_elapsed": 0.39083779399993546}, "coarsen_multipoles": {"wall_elapsed": 0.7719269678927958, "process_elapsed": 0.8146186569999827}, "eval_direct": {"wall_elapsed": 17.227728134952486, "process_elapsed": 17.294979483999896}, "multipole_to_local": {"wall_elapsed": 166.3015058990568, "process_elapsed": 165.78228093299992}, "eval_multipoles": {"wall_elapsed": 44.71846476290375, "process_elapsed": 44.62762825000004}, "form_locals": {"wall_elapsed": 9.726129946764559, "process_elapsed": 9.721420520000038}, "refine_locals": {"wall_elapsed": 0.5776261049322784, "process_elapsed": 0.5694936979999738}, "eval_locals": {"wall_elapsed": 0.3503082669340074, "process_elapsed": 0.350304264999977}}, {"nterms_fmm_total": 7000000, "direct_workload": 67754792, "direct_nsource_boxes": 399866, "m2l_workload": 7699296000, "m2p_workload": 1423135200, "m2p_nboxes": 1352554, "p2l_workload": 330677000, "p2l_nboxes": 356695, "eval_part_workload": 7000000, "form_multipoles": {"wall_elapsed": 0.29380203830078244, "process_elapsed": 0.29537681300007534}, "coarsen_multipoles": {"wall_elapsed": 0.6812492711469531, "process_elapsed": 0.7019599970000172}, "eval_direct": {"wall_elapsed": 26.609884928911924, "process_elapsed": 26.486387821999756}, "multipole_to_local": {"wall_elapsed": 225.8225390901789, "process_elapsed": 218.796968299}, "eval_multipoles": {"wall_elapsed": 72.68378263898194, "process_elapsed": 64.27706159500008}, "form_locals": {"wall_elapsed": 15.491025436203927, "process_elapsed": 15.374123016999874}, "refine_locals": {"wall_elapsed": 0.797229612711817, "process_elapsed": 0.7972753879998891}, "eval_locals": {"wall_elapsed": 0.7185860709287226, "process_elapsed": 0.6837079179999819}}, {"nterms_fmm_total": 8000000, "direct_workload": 82707417, "direct_nsource_boxes": 441394, "m2l_workload": 8617704000, "m2p_workload": 1685913200, "m2p_nboxes": 1498566, "p2l_workload": 348565200, "p2l_nboxes": 382521, "eval_part_workload": 8000000, "form_multipoles": {"wall_elapsed": 0.8264692649245262, "process_elapsed": 0.5747898250001526}, "coarsen_multipoles": {"wall_elapsed": 1.5497954818420112, "process_elapsed": 1.3431830110000647}, "eval_direct": {"wall_elapsed": 29.598126132041216, "process_elapsed": 29.095733828999755}, "multipole_to_local": {"wall_elapsed": 273.25275476509705, "process_elapsed": 266.123896437}, "eval_multipoles": {"wall_elapsed": 71.07263175630942, "process_elapsed": 70.09043602299994}, "form_locals": {"wall_elapsed": 14.648473134730011, "process_elapsed": 14.53982591099998}, "refine_locals": {"wall_elapsed": 1.0756985042244196, "process_elapsed": 1.0682339970001067}, "eval_locals": {"wall_elapsed": 0.7669271482154727, "process_elapsed": 0.6153591639999831}}, {"nterms_fmm_total": 1000000, "direct_workload": 5064570, "direct_nsource_boxes": 56907, "m2l_workload": 879645000, "m2p_workload": 80054000, "m2p_nboxes": 148660, "p2l_workload": 58602800, "p2l_nboxes": 84384, "eval_part_workload": 1000000, "form_multipoles": {"wall_elapsed": 0.09652932407334447, "process_elapsed": 0.09385259800000023}, "coarsen_multipoles": {"wall_elapsed": 0.1438053478486836, "process_elapsed": 0.14380348099999996}, "eval_direct": {"wall_elapsed": 2.8724248111248016, "process_elapsed": 2.8874125049999955}, "multipole_to_local": {"wall_elapsed": 22.150050774216652, "process_elapsed": 22.163823834000002}, "eval_multipoles": {"wall_elapsed": 4.035081175155938, "process_elapsed": 4.018481735999998}, "form_locals": {"wall_elapsed": 2.4109095097519457, "process_elapsed": 2.410357944000001}, "refine_locals": {"wall_elapsed": 0.10400902340188622, "process_elapsed": 0.10400039700000008}, "eval_locals": {"wall_elapsed": 0.05630338191986084, "process_elapsed": 0.05630333900000295}}, {"nterms_fmm_total": 2000000, "direct_workload": 14111527, "direct_nsource_boxes": 105337, "m2l_workload": 1702853000, "m2p_workload": 202584200, "m2p_nboxes": 311728, "p2l_workload": 120805600, "p2l_nboxes": 146698, "eval_part_workload": 2000000, "form_multipoles": {"wall_elapsed": 0.10583456791937351, "process_elapsed": 0.1058346049999983}, "coarsen_multipoles": {"wall_elapsed": 0.24514786386862397, "process_elapsed": 0.24514414600000123}, "eval_direct": {"wall_elapsed": 5.353780907113105, "process_elapsed": 5.363511694999993}, "multipole_to_local": {"wall_elapsed": 40.435692673083395, "process_elapsed": 40.370689201000005}, "eval_multipoles": {"wall_elapsed": 8.241205959115177, "process_elapsed": 8.222697027999999}, "form_locals": {"wall_elapsed": 4.570903809741139, "process_elapsed": 4.547761195999996}, "refine_locals": {"wall_elapsed": 0.22210092516615987, "process_elapsed": 0.22209859000000165}, "eval_locals": {"wall_elapsed": 0.13560375943779945, "process_elapsed": 0.13560390499999642}}, {"nterms_fmm_total": 3000000, "direct_workload": 24428758, "direct_nsource_boxes": 223172, "m2l_workload": 4011675000, "m2p_workload": 670713600, "m2p_nboxes": 735176, "p2l_workload": 213069300, "p2l_nboxes": 259667, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.22054569516330957, "process_elapsed": 0.22053314199999363}, "coarsen_multipoles": {"wall_elapsed": 0.5125908548943698, "process_elapsed": 0.5085106519999982}, "eval_direct": {"wall_elapsed": 12.12822123710066, "process_elapsed": 12.215636038999989}, "multipole_to_local": {"wall_elapsed": 93.70578746590763, "process_elapsed": 93.417041737}, "eval_multipoles": {"wall_elapsed": 25.345406110864133, "process_elapsed": 25.348133571999995}, "form_locals": {"wall_elapsed": 8.22137835714966, "process_elapsed": 8.202794989000012}, "refine_locals": {"wall_elapsed": 0.41789698833599687, "process_elapsed": 0.4185953409999854}, "eval_locals": {"wall_elapsed": 0.269101491663605, "process_elapsed": 0.26908321999999885}}, {"nterms_fmm_total": 4000000, "direct_workload": 29106936, "direct_nsource_boxes": 259688, "m2l_workload": 4679517000, "m2p_workload": 737745100, "m2p_nboxes": 887508, "p2l_workload": 262131700, "p2l_nboxes": 277715, "eval_part_workload": 4000000, "form_multipoles": {"wall_elapsed": 0.17082596709951758, "process_elapsed": 0.1708260939999775}, "coarsen_multipoles": {"wall_elapsed": 0.4154431517235935, "process_elapsed": 0.42217062099999225}, "eval_direct": {"wall_elapsed": 11.69108117558062, "process_elapsed": 11.81256067199999}, "multipole_to_local": {"wall_elapsed": 105.88159589888528, "process_elapsed": 105.51405778699998}, "eval_multipoles": {"wall_elapsed": 28.176122231874615, "process_elapsed": 28.119217985000034}, "form_locals": {"wall_elapsed": 8.196291028987616, "process_elapsed": 8.181508215999997}, "refine_locals": {"wall_elapsed": 0.46452207770198584, "process_elapsed": 0.4564281270000379}, "eval_locals": {"wall_elapsed": 0.25099264504387975, "process_elapsed": 0.25098168199997417}}, {"nterms_fmm_total": 5000000, "direct_workload": 47923958, "direct_nsource_boxes": 321217, "m2l_workload": 6098447000, "m2p_workload": 1049959000, "m2p_nboxes": 1058414, "p2l_workload": 250873800, "p2l_nboxes": 307399, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.27000816026702523, "process_elapsed": 0.2720453750000047}, "coarsen_multipoles": {"wall_elapsed": 0.518343870062381, "process_elapsed": 0.5315893110000047}, "eval_direct": {"wall_elapsed": 15.501696541905403, "process_elapsed": 15.713185632000034}, "multipole_to_local": {"wall_elapsed": 145.782555392012, "process_elapsed": 145.32088750699995}, "eval_multipoles": {"wall_elapsed": 38.39985727425665, "process_elapsed": 38.26757356000007}, "form_locals": {"wall_elapsed": 7.733311468735337, "process_elapsed": 7.705839346999937}, "refine_locals": {"wall_elapsed": 0.5397246889770031, "process_elapsed": 0.540322248999928}, "eval_locals": {"wall_elapsed": 0.32426714105531573, "process_elapsed": 0.3242635839999366}}, {"nterms_fmm_total": 6000000, "direct_workload": 52175740, "direct_nsource_boxes": 365978, "m2l_workload": 6963407000, "m2p_workload": 1228104100, "m2p_nboxes": 1238954, "p2l_workload": 320901400, "p2l_nboxes": 340870, "eval_part_workload": 6000000, "form_multipoles": {"wall_elapsed": 0.24512064224109054, "process_elapsed": 0.25092624599994906}, "coarsen_multipoles": {"wall_elapsed": 0.7491539157927036, "process_elapsed": 0.7398730170000363}, "eval_direct": {"wall_elapsed": 17.279178180731833, "process_elapsed": 17.591523772999835}, "multipole_to_local": {"wall_elapsed": 161.031417501159, "process_elapsed": 160.49498069599997}, "eval_multipoles": {"wall_elapsed": 45.24870359105989, "process_elapsed": 45.14435771700005}, "form_locals": {"wall_elapsed": 10.377049359958619, "process_elapsed": 10.342459472999963}, "refine_locals": {"wall_elapsed": 0.7589819608256221, "process_elapsed": 0.754924054000071}, "eval_locals": {"wall_elapsed": 0.4476486751809716, "process_elapsed": 0.4476429339999868}}, {"nterms_fmm_total": 7000000, "direct_workload": 67754792, "direct_nsource_boxes": 399866, "m2l_workload": 7699296000, "m2p_workload": 1423135200, "m2p_nboxes": 1352554, "p2l_workload": 330677000, "p2l_nboxes": 356695, "eval_part_workload": 7000000, "form_multipoles": {"wall_elapsed": 0.27594426879659295, "process_elapsed": 0.28047744100001637}, "coarsen_multipoles": {"wall_elapsed": 0.7236054469831288, "process_elapsed": 0.7297952440000017}, "eval_direct": {"wall_elapsed": 20.532363687176257, "process_elapsed": 20.546230066000135}, "multipole_to_local": {"wall_elapsed": 180.1073812278919, "process_elapsed": 179.505576249}, "eval_multipoles": {"wall_elapsed": 51.82399559998885, "process_elapsed": 51.64447850400006}, "form_locals": {"wall_elapsed": 13.09091425826773, "process_elapsed": 13.063926417999937}, "refine_locals": {"wall_elapsed": 0.6001745373941958, "process_elapsed": 0.600155986000118}, "eval_locals": {"wall_elapsed": 0.36805562302470207, "process_elapsed": 0.3680515439998544}}, {"nterms_fmm_total": 8000000, "direct_workload": 82707417, "direct_nsource_boxes": 441394, "m2l_workload": 8617704000, "m2p_workload": 1685913200, "m2p_nboxes": 1498566, "p2l_workload": 348565200, "p2l_nboxes": 382521, "eval_part_workload": 8000000, "form_multipoles": {"wall_elapsed": 0.39941675309091806, "process_elapsed": 0.3981763469998896}, "coarsen_multipoles": {"wall_elapsed": 0.9822220490314066, "process_elapsed": 1.0149941570000465}, "eval_direct": {"wall_elapsed": 21.01450498914346, "process_elapsed": 21.073397865999823}, "multipole_to_local": {"wall_elapsed": 197.67101062694564, "process_elapsed": 197.05391954499987}, "eval_multipoles": {"wall_elapsed": 58.286998888943344, "process_elapsed": 58.08302871900014}, "form_locals": {"wall_elapsed": 11.379268935415894, "process_elapsed": 11.344901745000016}, "refine_locals": {"wall_elapsed": 0.659439907874912, "process_elapsed": 0.6594181859998116}, "eval_locals": {"wall_elapsed": 0.48237756825983524, "process_elapsed": 0.47433469900011005}}]
\ No newline at end of file
diff --git a/test/test_distributed.py b/test/test_distributed.py
index aa50100..217374a 100644
--- a/test/test_distributed.py
+++ b/test/test_distributed.py
@@ -89,7 +89,7 @@ def _test_against_shared(dims, nsources, ntargets, dtype):
 
 @pytest.mark.mpi
 @pytest.mark.parametrize("num_processes, dims, nsources, ntargets", [
-    (4, 3, 10000, 10000)
+    (4, 3, 10, 10)
 ])
 def test_against_shared(num_processes, dims, nsources, ntargets):
     pytest.importorskip("mpi4py")
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment