GPU CMake: enable the Metal backend by default where the toolchain supports it

GPU/GPUTracking/CMakeLists.txt

@@ -513,9 +513,7 @@ if(CPUDEVICE_ENABLED)
 endif()
 
 if(ALIGPU_BUILD_TYPE STREQUAL "O2" OR ALIGPU_BUILD_TYPE STREQUAL "Standalone")
-  if(NOT ${CMAKE_SYSTEM} MATCHES Darwin)
-    add_subdirectory(Standalone/Benchmark)
-  endif()
+  add_subdirectory(Standalone/Benchmark)
 endif()
 
 if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_GPU})

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -348,8 +348,8 @@ AddOption(keepAllMemory, bool, false, "", 0, "Allocate all memory on both device
 AddOption(keepDisplayMemory, bool, false, "", 0, "Like keepAllMemory, but only for memory required for event display")
 AddOption(disableMemoryReuse, bool, false, "", 0, "Disable memory reusage (for debugging only)")
 AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strategy (0 = auto, 1 = individual allocations, 2 = single global allocation)")
-AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory (0 = disable, 1 / 2 = auto, other values set the size of the memory pool)", min(0ul))
-AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(0ul))
+AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory (0 = disable, 1 / 2 = auto, other values set the size of the memory pool)", min(uint64_t(0)))
+AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(uint64_t(0)))
 AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers")
 AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1))
 AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations")

GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h

@@ -47,7 +47,7 @@ EndConfig()
 BeginConfig(GPUSettingsStandalone, configStandalone)
 AddOption(runGPU, uint8_t, 1, "", 'g', "Use GPU for processing", message("GPU processing enabled"), set(2))
 AddOptionSet(runGPU, uint8_t, 0, "", 'c', "Use CPU for processing", message("CPU enabled"))
-AddOption(gpuType, std::string, "AUTO", "", 0, "GPU type (CUDA / HIP / OCL / CPUDEVICE) or CPU or AUTO")
+AddOption(gpuType, std::string, "AUTO", "", 0, "GPU type (CUDA / HIP / OCL / METAL / CPUDEVICE) or CPU or AUTO")
 AddOption(runGPUforce, bool, true, "", 0, "Force usage of the specified GPU device type, no CPU fallback")
 AddOption(promptOnExit, bool, false, "", 0, "Prompt for keypress before exiting")
 AddOption(continueOnError, bool, false, "", 0, "Continue processing after an error")
@@ -64,8 +64,8 @@ AddOption(eventDisplay, int32_t, 0, "display", 'd', "Show standalone event displ
 AddOption(eventGenerator, bool, false, "", 0, "Run event generator")
 AddOption(cont, bool, false, "", 0, "Process continuous timeframe data, even if input is triggered")
 AddOption(setMaxTimeBin, int32_t, -2, "", 0, "maximum time bin of continuous data, 0 for triggered events, -1 for automatic continuous mode, -2 for automatic continuous / triggered")
-AddOption(outputcontrolmem, uint64_t, 0, "outputMemory", 0, "Use predefined output buffer of this size", min(0ul), message("Using %s bytes as output memory"))
-AddOption(inputcontrolmem, uint64_t, 0, "inputMemory", 0, "Use predefined input buffer of this size", min(0ul), message("Using %s bytes as input memory"))
+AddOption(outputcontrolmem, uint64_t, 0, "outputMemory", 0, "Use predefined output buffer of this size", min(uint64_t(0)), message("Using %s bytes as output memory"))
+AddOption(inputcontrolmem, uint64_t, 0, "inputMemory", 0, "Use predefined input buffer of this size", min(uint64_t(0)), message("Using %s bytes as input memory"))
 AddOption(cpuAffinity, int32_t, -1, "", 0, "Pin CPU affinity to this CPU core", min(-1))
 AddOption(fifoScheduler, bool, false, "", 0, "Use FIFO realtime scheduler", message("Setting FIFO scheduler: %s"))
 AddOption(fpe, int8_t, -1, "", 0, "Trap on floating point exceptions (-1 = if no ffast-math)")

GPU/GPUTracking/Standalone/Benchmark/standalone.cxx

@@ -124,6 +124,7 @@ int32_t ReadConfiguration(int argc, char** argv)
   setlocale(LC_ALL, "en_US.utf-8");
   setlocale(LC_NUMERIC, "en_US.utf-8");
   if (configStandalone.cpuAffinity != -1) {
+#ifdef __linux__
     cpu_set_t mask;
     CPU_ZERO(&mask);
     CPU_SET(configStandalone.cpuAffinity, &mask);
@@ -133,8 +134,13 @@ int32_t ReadConfiguration(int argc, char** argv)
       printf("Error setting CPU affinity\n");
       return 1;
     }
+#else
+    printf("CPU affinity not supported on this platform\n");
+    return 1;
+#endif
   }
   if (configStandalone.fifoScheduler) {
+#ifdef __linux__
     printf("Setting FIFO scheduler\n");
     sched_param param;
     sched_getparam(0, &param);
@@ -143,13 +149,21 @@ int32_t ReadConfiguration(int argc, char** argv)
       printf("Error setting scheduler\n");
       return 1;
     }
+#else
+    printf("FIFO scheduler not supported on this platform\n");
+    return 1;
+#endif
   }
 #ifdef __FAST_MATH__
   if (configStandalone.fpe == 1) {
 #else
   if (configStandalone.fpe) {
 #endif
+#ifdef __linux__
     feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW);
+#else
+    printf("Trapping floating point exceptions not supported on this platform\n");
+#endif
   }
   bool detMode = false, noFTZMode = false;
 #ifdef GPUCA_DETERMINISTIC_MODE
@@ -1021,7 +1035,9 @@ int32_t main(int argc, char** argv)
 
 #ifndef _WIN32
   if (configStandalone.proc.runQA && configStandalone.fpe) {
+#ifdef __linux__
     fedisableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW);
+#endif
   }
 #endif
 

GPU/documentation/build-standalone.md

@@ -81,18 +81,42 @@ The standalone benchmark supports running on Run2 data exported from AliRoot, or
 In o2, `o2-tpc-reco-workflow` and the `o2-gpu-reco-workflow` can dump event data with the `configKeyValue` `GPU_global.dump=1;`.
 This will dump the event data to the local folder, all dumped files have a `.dump` file extension. If there are multiple TFs/events processed, there will be multiple `event.i.dump` files. In order to create a standalone dataset out of these, just copy all the `.dump` files to a subfolder in `~/standalone/events/[FOLDERNAME]`.
 
+## What a dataset consists of
+
+A dataset folder is read by `GPUReconstruction::ReadSettings` and `GPUChainTracking::ReadSettings`, and holds:
+
+- `settings.dump` - the GRP settings (solenoid field, continuous max time bin). Without it the benchmark aborts with `Error reading event config file`.
+- `event.<i>.dump` - one file per time frame / event.
+- the calibration objects, each written only if it was present when dumping: `tpctransform.dump`, `tpcpadgaincalib.dump`, `tpczslinkmapping.dump`, `dEdxCalibContainer.dump`, `matlut.dump`, `trdgeometry.dump`, `trdrecoparam.dump`.
+
+`tpctransform.dump` is needed for everything except already-transformed clusters; without it the run stops at `Need cluster native data for on-the-fly TPC transform`. `dEdxCalibContainer.dump` is needed unless dE/dx is switched off with `--rundEdx 0`.
+
+The folder is looked up under `./events/` relative to the working directory, so `-e mydata` reads `./events/mydata/`. Use `--absoluteEventsDir` to pass a path outside of `./events/`.
+
+## From simulated data
+
 Data can be dumped from raw data, or from MC data, e.g. generated by the Full System Test. In case of MC data, also MC labels are dumped, such that they are used in the `./ca --qa` mode.
 
-To get a dump from simulated data, please run e.g. the FST simulation as described in [full-system-test-setup.md](https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/full-system-test-setup.md).
-A simple run as
+First simulate, digitize and convert to raw, as described in [full-system-test-setup.md](https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/full-system-test-setup.md). Run it in a scratch folder, since it writes a lot of intermediate files:
 ```
-DISABLE_PROCESSING=1 NEvents=5 NEventsQED=100 SHMSIZE=16000000000 $O2_ROOT/prodtests/full_system_test.sh
+mkdir -p test && cd test
+ulimit -n 1024
+NEvents=5 NEventsQED=100 $O2_ROOT/prodtests/full_system_test.sh
 ```
-should be enough.
+`ulimit -n 1024` avoids the workflow running out of file descriptors. `NEventsQED` controls the QED background, which dominates the TPC occupancy, so lower it for a smaller dataset.
 
-Afterwards run the following command to dump the data:
+This leaves the raw time frames in `raw/` and already runs the reconstruction once. To get the dumps, re-run only the reconstruction over that raw data with the dump enabled:
 ```
-SYNCMODE=1 CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow="GPU_global.dump=1;" WORKFLOW_DETECTORS=TPC SHMSIZE=16000000000 $O2_ROOT/prodtests/full-system-test/dpl-workflow.sh
+ARGS_ALL_EXTRA="--completion-policy=wait --exit-transition-timeout 20" \
+  CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow="GPU_global.dump=1;"      \
+  WORKFLOW_DETECTORS=TPC SYNCMODE=1 SHMSIZE=$((16<<30)) EXTINPUT=1    \
+  $O2_ROOT/prodtests/full-system-test/dpl-workflow.sh
+```
+`CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow="GPU_global.dump=1;"` is what writes the `.dump` files; everything else just shapes the workflow. `--completion-policy=wait` and `--exit-transition-timeout 20` let the workflow drain and shut down by itself once the input ends, instead of having to be killed. `EXTINPUT=1` takes the raw data from the `readout-proxy` FMQ channel, which means a feeder has to run in a second shell (`datadistribution.sh` from the same folder, which needs the `DataDistribution` package). Leave `EXTINPUT` unset to have `dpl-workflow.sh` read the raw files directly instead, which needs no second process. See [env-variables.md](https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/env-variables.md) for the other variables.
+
+Finally collect the result into a dataset:
+```
+mkdir -p ~/standalone/events/fst-tpc && cp *.dump ~/standalone/events/fst-tpc/
 ```
 
 To dump standalone data from CTF raw data in `myctf.root`, you can use the same script, e.g.:
@@ -101,3 +125,16 @@ CTFINPUT=1 INPUT_FILE_LIST=myctf.root CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow=
 ```
 
 On the EPNs, you can find some reference data sets at `/home/drohr/standalone/events`.
+
+## Using the built-in event generator
+
+With `--eventGenerator` the benchmark generates simple synthetic events itself, without any simulation or raw data. The parent `events/` folder has to exist, since the generator only creates the dataset folder itself:
+```
+mkdir -p events
+./ca --eventGenerator -e gen --setMaxTimeBin 0 --rundEdx 0
+```
+`--setMaxTimeBin` has to be given, because the GRP settings are not read from a dataset in this mode and their default is the invalid sentinel `-2`.
+
+This writes `settings.dump` and the `event.<i>.dump` files, but no calibration objects. The events hold the legacy cluster format, not native clusters, so they can be inspected and used for the generator's own QA, but tracking on them stops at `Need cluster native data for on-the-fly TPC transform`, also with calibration objects present.
+
+`--recreateTrivialCalibObjects` writes default calibration objects (`tpctransform.dump`, `dEdxCalibContainer.dump`, ...) to the current folder without CCDB access; it needs `O2_ROOT` set for the TPC mapping files.

dependencies/FindO2GPU.cmake

@@ -10,7 +10,7 @@
 # or submit itself to any jurisdiction.
 
 # NOTE!!!! - Whenever this file is changed, move it over to alidist/resources
-# FindO2GPU.cmake Version 23
+# FindO2GPU.cmake Version 25
 
 set(CUDA_COMPUTETARGET_DEFAULT_FULL 80-real;86-real;89-real;120-real;75-virtual)
 set(HIP_AMDGPUTARGET_DEFAULT_FULL gfx906;gfx908)
@@ -21,8 +21,7 @@ if(NOT DEFINED ENABLE_CUDA)
   set(ENABLE_CUDA "AUTO")
 endif()
 if(NOT DEFINED ENABLE_METAL)
-  # OFF rather than AUTO while the backend is unproven
-  set(ENABLE_METAL "OFF")
+  set(ENABLE_METAL "AUTO")
 endif()
 if(NOT DEFINED ENABLE_OPENCL)
   set(ENABLE_OPENCL "AUTO")
@@ -467,15 +466,14 @@ if(ENABLE_METAL)
   # The frameworks are there on every macOS, but the backend needs MSL 4.1, which
   # is macOS 27 and its toolchain, so ask the compiler instead of assuming. A
   # library built as MSL 4.1 also only loads on macOS 27 and later.
-  if(NOT DEFINED GPUCA_METAL_MSL41)
+  # Only a success is cached, so that installing the toolchain is picked up.
+  if(NOT GPUCA_METAL_MSL41)
     set(GPUCA_METAL_PROBE "${CMAKE_CURRENT_BINARY_DIR}/metal_msl41_probe.metal")
     file(WRITE "${GPUCA_METAL_PROBE}" "#include <metal_stdlib>\nkernel void probe(device float* o [[buffer(0)]], uint i [[thread_position_in_grid]]) { o[i] = o[i] * 2.0f; }\n")
     execute_process(COMMAND xcrun -sdk macosx metal -std=metal4.1 -c "${GPUCA_METAL_PROBE}" -o "${GPUCA_METAL_PROBE}.air"
-                    RESULT_VARIABLE GPUCA_METAL_PROBE_RESULT OUTPUT_QUIET ERROR_QUIET)
+                    RESULT_VARIABLE GPUCA_METAL_PROBE_RESULT OUTPUT_QUIET ERROR_VARIABLE GPUCA_METAL_PROBE_ERROR)
     if(GPUCA_METAL_PROBE_RESULT EQUAL 0)
       set(GPUCA_METAL_MSL41 ON CACHE INTERNAL "Metal toolchain compiles MSL 4.1")
-    else()
-      set(GPUCA_METAL_MSL41 OFF CACHE INTERNAL "Metal toolchain compiles MSL 4.1")
     endif()
   endif()
   if(METAL_FRAMEWORK AND COREFOUNDATION_FRAMEWORK AND FOUNDATION_FRAMEWORK AND QUARTZCORE_FRAMEWORK AND GPUCA_METAL_MSL41)
@@ -487,8 +485,9 @@ if(ENABLE_METAL)
     if(NOT METAL_FRAMEWORK OR NOT COREFOUNDATION_FRAMEWORK OR NOT FOUNDATION_FRAMEWORK OR NOT QUARTZCORE_FRAMEWORK)
       message(FATAL_ERROR "Metal frameworks not available")
     endif()
-    message(FATAL_ERROR "The Metal backend needs MSL 4.1: this toolchain rejects -std=metal4.1, and the result would need macOS 27 to run")
+    message(FATAL_ERROR "The Metal backend needs MSL 4.1: this toolchain rejects -std=metal4.1, and the result would need macOS 27 to run\n${GPUCA_METAL_PROBE_ERROR}")
   else()
+    message(STATUS "Metal backend disabled, the toolchain does not compile MSL 4.1: ${GPUCA_METAL_PROBE_ERROR}")
     set(METAL_ENABLED OFF)
   endif()
 endif()

prodtests/full_system_test.sh

@@ -48,6 +48,7 @@ NJOBS=${NJOBS:-"${NCPUS}"}
 SHMSIZE=${SHMSIZE:-8000000000} # Size of shared memory for messages (use 128 GB for 550 event full TF)
 #TPCTRACKERSCRATCHMEMORY=${TPCTRACKERSCRATCHMEMORY:-8000000000} # Not needed by default
 ENABLE_GPU_TEST=${ENABLE_GPU_TEST:-0} # Run the full system test also on the GPU
+FST_GPUTYPE=${FST_GPUTYPE:-CUDA} # GPU backend used in case ENABLE_GPU_TEST=1 (CUDA / HIP / METAL)
 GPUMEMSIZE=${GPUMEMSIZE:-6000000000} # Size of GPU memory to use in case ENABBLE_GPU_TEST=1
 NTIMEFRAMES=${NTIMEFRAMES:-1} # Number of time frames to process
 TFDELAY=${TFDELAY:-100} # Delay in seconds between publishing time frames
@@ -281,7 +282,7 @@ for STAGE in $STAGES; do
   export WORKFLOW_PARAMETERS=
   if [[ "$STAGE" = "WITHGPU" ]]; then
     export CREATECTFDICT=0
-    export GPUTYPE=CUDA
+    export GPUTYPE=$FST_GPUTYPE
     export HOSTMEMSIZE=1000000000
     export SYNCMODE=1
     export CTFINPUT=0