diff --git a/.github/workflows/ubuntu-cache-idioms.yml b/.github/workflows/ubuntu-cache-idioms.yml new file mode 100644 index 000000000..2aa434810 --- /dev/null +++ b/.github/workflows/ubuntu-cache-idioms.yml @@ -0,0 +1,31 @@ +name: Ubuntu (cache,IDIOMS) + +on: + pull_request: + branches: [ stable, develop ] + + push: + branches: [ stable, develop ] + + workflow_dispatch: + +jobs: + PDC: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v3 + + - name: Dependencies + run: .github/workflows/dependencies-linux.sh + + - name: Build PDC with IDIOMS + run: | + mkdir build && cd build + cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_ENABLE_IDIOMS=ON -DPDC_SERVER_CACHE=ON -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + make -j2 + + - name: Test PDC + working-directory: build + run: ctest -L serial --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index dfeb7dc00..4bb931a3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,12 +328,18 @@ if(PDC_USE_CRAY_DRC) endif() #----------------------------------------------------------------------------- -# DART Suffix Tree mode +# IDIOMS Index #----------------------------------------------------------------------------- -option(PDC_DART_SUFFIX_TREE_MODE "Enable DART Suffix Tree mode." ON) -if(PDC_DART_SUFFIX_TREE_MODE) - set(PDC_DART_SFX_TREE 1) - # add_compile_definitions(PDC_DART_SFX_TREE=${PDC_DART_SFX_TREE}) +option(PDC_ENABLE_IDIOMS "Enable IDIOMS metadata." OFF) +if(PDC_ENABLE_IDIOMS) + set(PDC_ENABLE_IDIOMS 1) + option(PDC_DART_SUFFIX_TREE_MODE "Enable DART Suffix Tree mode." ON) + if(PDC_DART_SUFFIX_TREE_MODE) + set(PDC_DART_SFX_TREE 1) + # add_compile_definitions(PDC_DART_SFX_TREE=${PDC_DART_SFX_TREE}) +endif() + + endif() #----------------------------------------------------------------------------- diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index ca2cb7ff2..060601d5f 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -1255,17 +1255,19 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po #endif if ((hg_transport = getenv("HG_TRANSPORT")) == NULL) { - LOG_INFO("Environment variable HG_TRANSPORT was NOT set\n"); hg_transport = default_hg_transport; + if (pdc_client_mpi_rank_g == 0) + LOG_INFO("Environment variable HG_TRANSPORT was NOT set, default to %s\n", default_hg_transport); } else LOG_INFO("Environment variable HG_TRANSPORT was set\n"); if ((hostname = getenv("HG_HOST")) == NULL) { - LOG_INFO("Environment variable HG_HOST was NOT set\n"); hostname = PDC_malloc(HOSTNAME_LEN); memset(hostname, 0, HOSTNAME_LEN); gethostname(hostname, HOSTNAME_LEN - 1); free_hostname = true; + if (pdc_client_mpi_rank_g == 0) + LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); } else LOG_INFO("Environment variable HG_HOST was set\n"); diff --git a/src/api/pdc_region/pdc_region_transfer.c b/src/api/pdc_region/pdc_region_transfer.c index 539ed1d52..dda936f02 100644 --- a/src/api/pdc_region/pdc_region_transfer.c +++ b/src/api/pdc_region/pdc_region_transfer.c @@ -855,43 +855,42 @@ register_metadata(pdc_transfer_request_start_all_pkg **transfer_request_input, i remain_size = input_size - size; output_size = 0; - index = 0; qsort(transfer_requests, size, sizeof(pdc_transfer_request_start_all_pkg *), sort_by_metadata_server_start_all); - for (i = 1; i < size; ++i) { - if (transfer_requests[i]->transfer_request->metadata_server_id != - transfer_requests[i - 1]->transfer_request->metadata_server_id) { - n_objs = i - index; - pack_region_metadata_query(transfer_requests + index, n_objs, &buf, &total_buf_size); - PDC_Client_transfer_request_metadata_query( - &bulk_handle, buf, total_buf_size, n_objs, - transfer_requests[index]->transfer_request->metadata_server_id, is_write, &output_buf_size, - &query_id); - PDCregion_transfer_add_bulk_handle(transfer_requests[index]->transfer_request, bulk_handle); - buf = (char *)PDC_free(buf); - if (query_id) { - output_buf = (char *)PDC_malloc(output_buf_size); - PDC_Client_transfer_request_metadata_query2( - &bulk_handle, output_buf, output_buf_size, query_id, - transfer_requests[index]->transfer_request->metadata_server_id); - PDCregion_transfer_add_bulk_handle(transfer_requests[index]->transfer_request, bulk_handle); - unpack_region_metadata_query(output_buf, transfer_requests + index, &transfer_request_head, - &transfer_request_end, &output_size); - output_buf = (char *)PDC_free(output_buf); - if (transfer_request_front_head) { - previous->next = transfer_request_head; - } - else { - transfer_request_front_head = transfer_request_head; - } - previous = transfer_request_end; - } - index = i; + + // Each iteration finds the first transfer that has a target meta server different from the previous one + // index is the first transfer index + int current_unique_idx = 0; + int *unique_server_xfer_idx = NULL; + int *unique_server_nboj = NULL; + if (size > 0) { + unique_server_xfer_idx = (int *)PDC_calloc(size, sizeof(int)); + unique_server_nboj = (int *)PDC_calloc(size, sizeof(int)); + } + + // Iterate through the input array + for (i = 0; i < size; ++i) { + if (i == 0 || transfer_requests[i]->transfer_request->metadata_server_id != + transfer_requests[i - 1]->transfer_request->metadata_server_id) { + // Check if the current element is different from the previous one + // or if it's the first element + unique_server_xfer_idx[current_unique_idx] = i; + unique_server_nboj[current_unique_idx] = 1; + + current_unique_idx++; + } + else { + unique_server_nboj[current_unique_idx - 1]++; } } + int num_unique_server_ids = current_unique_idx; + + // Now we will try to distribute the metadata requests to different servers across clients + for (i = 0; i < num_unique_server_ids; i++) { + int current_index = (pdc_client_mpi_rank_g + i) % num_unique_server_ids; + index = unique_server_xfer_idx[current_index]; + n_objs = unique_server_nboj[current_index]; - if (size) { - n_objs = size - index; pack_region_metadata_query(transfer_requests + index, n_objs, &buf, &total_buf_size); PDC_Client_transfer_request_metadata_query( &bulk_handle, buf, total_buf_size, n_objs, @@ -899,7 +898,6 @@ register_metadata(pdc_transfer_request_start_all_pkg **transfer_request_input, i &query_id); PDCregion_transfer_add_bulk_handle(transfer_requests[index]->transfer_request, bulk_handle); buf = (char *)PDC_free(buf); - // If it is a valid query ID, then it means regions are overlapping. if (query_id) { output_buf = (char *)PDC_malloc(output_buf_size); PDC_Client_transfer_request_metadata_query2( @@ -909,16 +907,21 @@ register_metadata(pdc_transfer_request_start_all_pkg **transfer_request_input, i unpack_region_metadata_query(output_buf, transfer_requests + index, &transfer_request_head, &transfer_request_end, &output_size); output_buf = (char *)PDC_free(output_buf); - if (transfer_request_front_head) { + + if (transfer_request_front_head) previous->next = transfer_request_head; - } - else { + else transfer_request_front_head = transfer_request_head; - } + previous = transfer_request_end; } } + if (unique_server_xfer_idx) + free(unique_server_xfer_idx); + if (unique_server_nboj) + free(unique_server_nboj); + if (output_size) { transfer_request_output = (pdc_transfer_request_start_all_pkg **)PDC_malloc( sizeof(pdc_transfer_request_start_all_pkg *) * (output_size + remain_size)); diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 9700d8628..c1e7bb0d5 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -23,6 +23,9 @@ if(PDC_ENABLE_ZFP) # find_path(ZFP_INCLUDE_DIR include/zfp.h) endif() +if(PDC_ENABLE_IDIOMS) + add_definitions(-DPDC_ENABLE_IDIOMS=1) +endif() include_directories( ${PDC_COMMON_INCLUDE_DIRS} diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index ffb967dbd..abe35434d 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -774,17 +774,19 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) total_mem_usage_g += (sizeof(char) + sizeof(char *)); if ((hg_transport = getenv("HG_TRANSPORT")) == NULL) { - LOG_INFO("Environment variable HG_TRANSPORT was NOT set\n"); hg_transport = default_hg_transport; + if (pdc_server_rank_g == 0) + LOG_INFO("Environment variable HG_TRANSPORT was NOT set, default to %s\n", hg_transport); } else LOG_INFO("Environment variable HG_TRANSPORT was set\n"); if ((hostname = getenv("HG_HOST")) == NULL) { - LOG_INFO("Environment variable HG_HOST was NOT set\n"); hostname = PDC_malloc(HOSTNAME_LEN); memset(hostname, 0, HOSTNAME_LEN); gethostname(hostname, HOSTNAME_LEN - 1); free_hostname = true; + if (pdc_server_rank_g == 0) + LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); } else LOG_INFO("Environment variable HG_HOST was set\n"); @@ -922,8 +924,12 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) LOG_INFO("Read cache enabled\n"); #endif +#ifdef PDC_ENABLE_IDIOMS // Initialize IDIOMS + if (pdc_server_rank_g == 0) + LOG_INFO("IDIOMS index enabled\n"); PDC_Server_metadata_index_init(pdc_server_size_g, pdc_server_rank_g); +#endif // TODO: support restart with different number of servers than previous run char checkpoint_file[ADDR_MAX + sizeof(int) + 1]; @@ -934,7 +940,9 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) ret_value = PDC_Server_restart(checkpoint_file); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "Error with PDC_Server_restart"); +#ifdef PDC_ENABLE_IDIOMS metadata_index_recover(pdc_server_tmp_dir_g, pdc_server_size_g, pdc_server_rank_g); +#endif } else { // We are starting a brand new server @@ -1364,7 +1372,9 @@ PDC_Server_checkpoint() LOG_INFO("Checkpointed %10d objects, with %10d regions \n", all_metadata_size, all_region_count); } +#ifdef PDC_ENABLE_IDIOMS metadata_index_dump(pdc_server_tmp_dir_g, pdc_server_rank_g); +#endif done: FUNC_LEAVE(ret_value); @@ -1417,11 +1427,6 @@ PDC_Server_restart(char *filename) if (file == NULL) PGOTO_ERROR(FAIL, "Error with fopen, filename: [%s]", filename); - char *slurm_jobid = getenv("SLURM_JOB_ID"); - if (slurm_jobid == NULL) { - LOG_ERROR("Error getting slurm job id from SLURM_JOB_ID\n"); - } - if (fread(&n_cont, sizeof(int), 1, file) != 1) { LOG_ERROR("Read failed for n_count\n"); } @@ -1971,8 +1976,10 @@ PDC_Server_mercury_register() PDC_region_analysis_release_register(hg_class_g); // DART Index +#ifdef PDC_ENABLE_IDIOMS PDC_dart_get_server_info_register(hg_class_g); PDC_dart_perform_one_server_register(hg_class_g); +#endif // Server to client RPC server_lookup_client_register_id_g = PDC_server_lookup_client_register(hg_class_g); diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index bf9935c84..40cebf5f9 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -80,6 +80,7 @@ set(PROGRAMS misc/vpicio misc/vpicio_mts misc/bdcats + misc/bdcats_mts deprecated/vpicio_old deprecated/bdcats_old tags/kvtag_add_get @@ -139,9 +140,14 @@ if(BUILD_MPI_TESTING) tags/kvtag_affix_query_scale tags/kvtag_add_get_benchmark tags/kvtag_add_get_scale + ) + +if(PDC_ENABLE_IDIOMS) + list(APPEND MPI_PROGRAMS misc/llsm_idioms_bench misc/idioms_ci_test ) +endif() foreach(program ${MPI_PROGRAMS}) get_filename_component(program_name ${program} NAME) @@ -261,7 +267,9 @@ add_test(NAME obj_buf WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTO add_test(NAME obj_tags WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_tags ) add_test(NAME kvtag_add_get WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./kvtag_add_get) add_test(NAME kvtag_query WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./kvtag_query 100 1 10 0) +if(PDC_ENABLE_IDIOMS) add_test(NAME idioms_ci_test WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./idioms_ci_test) +endif() add_test(NAME obj_info WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_info ) add_test(NAME obj_put_data WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_put_data ) add_test(NAME obj_get_data WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_get_data ) @@ -332,7 +340,9 @@ set_tests_properties(obj_buf PROPERTIES LABELS serial) set_tests_properties(obj_tags PROPERTIES LABELS serial) set_tests_properties(kvtag_add_get PROPERTIES LABELS serial) set_tests_properties(kvtag_query PROPERTIES LABELS serial) +if(PDC_ENABLE_IDIOMS) set_tests_properties(idioms_ci_test PROPERTIES LABELS serial) +endif() set_tests_properties(obj_info PROPERTIES LABELS serial) set_tests_properties(obj_put_data PROPERTIES LABELS serial) set_tests_properties(obj_get_data PROPERTIES LABELS serial) diff --git a/src/tests/misc/bdcats_mts.c b/src/tests/misc/bdcats_mts.c new file mode 100644 index 000000000..d3088d9a5 --- /dev/null +++ b/src/tests/misc/bdcats_mts.c @@ -0,0 +1,272 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_timing.h" + +#define NPARTICLES 8388608 + +double +uniform_random_number() +{ + return (((double)rand()) / ((double)(RAND_MAX))); +} + +void +print_usage() +{ + LOG_JUST_PRINT("Usage: srun -n ./bdcats_mts #particles #steps sleep_time(s)\n"); +} + +int +main(int argc, char **argv) +{ + int rank = 0, size = 1; + pdcid_t pdc_id, cont_id, region_local, region_remote; + pdcid_t obj_ids[8]; +#ifdef ENABLE_MPI + MPI_Comm comm; +#else + int comm = 1; +#endif + float * dx, *dy, *dz, *ux, *uy, *uz, *q; + int * id; + int x_dim = 64, y_dim = 64, z_dim = 64, ndim = 1, steps = 1, sleeptime = 0; + uint64_t numparticles, dims[1], offset_local[1], offset_remote[1], mysize[1]; + double t0, t1; + const char *obj_names[] = {"dX", "dY", "dZ", "Ux", "Uy", "Uz", "q", "i"}; + char obj_name[64]; + + pdcid_t transfer_requests[8]; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + MPI_Comm_dup(MPI_COMM_WORLD, &comm); +#endif + + numparticles = NPARTICLES; + if (argc == 4) { + numparticles = atoll(argv[1]); + steps = atoi(argv[2]); + sleeptime = atoi(argv[3]); + } + if (rank == 0) + LOG_INFO("Writing %" PRIu64 " number of particles for %d steps with %d clients.\n", numparticles, + steps, size); + + dims[0] = numparticles * size; + + dx = (float *)malloc(numparticles * sizeof(float)); + dy = (float *)malloc(numparticles * sizeof(float)); + dz = (float *)malloc(numparticles * sizeof(float)); + ux = (float *)malloc(numparticles * sizeof(float)); + uy = (float *)malloc(numparticles * sizeof(float)); + uz = (float *)malloc(numparticles * sizeof(float)); + q = (float *)malloc(numparticles * sizeof(float)); + + id = (int *)malloc(numparticles * sizeof(int)); + + void *data_ptrs[] = {&dx[0], &dy[0], &dz[0], &ux[0], &uy[0], &uz[0], &id[0], &q[0]}; + + // create a pdc + pdc_id = PDCinit("pdc"); + + // create a container + cont_id = PDCcont_open("c1", pdc_id); + if (cont_id <= 0) { + LOG_ERROR("Failed to create container"); + return FAIL; + } + + offset_local[0] = 0; + offset_remote[0] = rank * numparticles; + mysize[0] = numparticles; + + // create local and remote region + region_local = PDCregion_create(ndim, offset_local, mysize); + region_remote = PDCregion_create(ndim, offset_remote, mysize); + + for (int iter = 0; iter < steps; iter++) { + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + if (rank == 0) + LOG_INFO("\n#Step %d\n", iter); + t0 = MPI_Wtime(); +#endif + for (int i = 0; i < 8; i++) { + sprintf(obj_name, "%s-%d", obj_names[i], iter); + obj_ids[i] = PDCobj_open(obj_name, pdc_id); + if (obj_ids[i] == 0) { + LOG_ERROR("Error getting an object id of %s from server\n", obj_name); + return FAIL; + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + t1 = MPI_Wtime(); + if (rank == 0) + LOG_INFO("Obj open time: %.5e\n", t1 - t0); +#endif + + for (int i = 0; i < 8; i++) { + transfer_requests[i] = + PDCregion_transfer_create(data_ptrs[i], PDC_READ, obj_ids[i], region_local, region_remote); + if (transfer_requests[i] == 0) { + LOG_ERROR("%s transfer request creation failed\n", obj_names[i]); + return FAIL; + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); + if (rank == 0) + LOG_INFO("Transfer create time: %.5e\n", t0 - t1); +#endif + +#ifdef ENABLE_MPI + if (PDCregion_transfer_start_all_mpi(transfer_requests, 8, MPI_COMM_WORLD) != SUCCEED) { +#else + if (PDCregion_transfer_start_all(transfer_requests, 8) != SUCCEED) { +#endif + LOG_ERROR("Failed to start transfer requests\n"); + return FAIL; + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + t1 = MPI_Wtime(); + if (rank == 0) + LOG_INFO("Transfer start time: %.5e\n", t1 - t0); +#endif + // Emulate compute with sleep + if (iter != steps - 1) { + if (rank == 0) + LOG_INFO("Sleep start: %llu.00\n", sleeptime); + sleep(sleeptime); + if (rank == 0) + LOG_INFO("Sleep end: %llu.00\n", sleeptime); + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); +#endif + + if (PDCregion_transfer_wait_all(transfer_requests, 8) != SUCCEED) { + LOG_ERROR("Failed to transfer wait all\n"); + return FAIL; + } + + // Verify data of id and q + if (id[0] != rank + iter || q[0] != rank + iter * 2 || id[numparticles - 1] != rank - iter || + q[numparticles - 1] != rank - iter * 2) { + LOG_ERROR("Data verification failed on rank %d for step %d! id[0]=%d/%d, " + "q[0]=%d/%d, id[end]=%d/%d, q[end]=%d/%d\n", + rank, iter, id[0], rank + iter, q[0], rank + iter * 2, id[numparticles - 1], + rank - iter, q[numparticles - 1], rank - iter * 2); + return FAIL; + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + t1 = MPI_Wtime(); + if (rank == 0) + LOG_INFO("Transfer wait time: %.5e\n", t1 - t0); +#endif + + for (int j = 0; j < 8; j++) { + if (PDCregion_transfer_close(transfer_requests[j]) != SUCCEED) { + LOG_ERROR("region transfer close failed\n"); + return FAIL; + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); + if (rank == 0) + LOG_INFO("Transfer close time: %.5e\n", t0 - t1); +#endif + + for (int i = 0; i < 8; i++) { + if (PDCobj_close(obj_ids[i]) != SUCCEED) { + LOG_ERROR("Failed to close object #%d\n", i); + return FAIL; + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + t1 = MPI_Wtime(); + if (rank == 0) + LOG_INFO("Obj close time: %.5e\n", t1 - t0); +#endif + } // End for steps + + PDC_timing_report("write"); + + if (PDCregion_close(region_local) != SUCCEED) { + LOG_ERROR("Failed to close local region \n"); + return FAIL; + } + if (PDCobj_close(region_remote) != SUCCEED) { + LOG_ERROR("Failed to close remote region\n"); + return FAIL; + } + if (PDCcont_close(cont_id) != SUCCEED) { + LOG_ERROR("Failed to close container\n"); + return FAIL; + } + if (PDCclose(pdc_id) != SUCCEED) { + LOG_ERROR("Failed to close PDC\n"); + return FAIL; + } + free(dx); + free(dy); + free(dz); + free(ux); + free(uy); + free(uz); + free(q); + free(id); +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return 0; +} diff --git a/src/tests/misc/vpicio_mts.c b/src/tests/misc/vpicio_mts.c index dff272985..18b6c9ff6 100644 --- a/src/tests/misc/vpicio_mts.c +++ b/src/tests/misc/vpicio_mts.c @@ -54,23 +54,20 @@ main(int argc, char **argv) int rank = 0, size = 1; pdcid_t pdc_id, cont_prop, cont_id, region_local, region_remote; pdcid_t obj_prop_float, obj_prop_int; - pdcid_t obj_xx, obj_yy, obj_zz, obj_pxx, obj_pyy, obj_pzz, obj_id11, obj_id22; + pdcid_t obj_ids[8]; #ifdef ENABLE_MPI MPI_Comm comm; #else int comm = 1; #endif - float * x, *y, *z, *px, *py, *pz; - int * id1, *id2; - int x_dim = 64, y_dim = 64, z_dim = 64, ndim = 1, steps = 1, sleeptime = 0; - uint64_t numparticles, dims[1], offset_local[1], offset_remote[1], mysize[1]; - double t0, t1; - char cur_time[64]; - time_t t; - struct tm *tm; - - pdcid_t transfer_request_x, transfer_request_y, transfer_request_z, transfer_request_px, - transfer_request_py, transfer_request_pz, transfer_request_id1, transfer_request_id2; + float * dx, *dy, *dz, *ux, *uy, *uz, *q; + int * id; + int x_dim = 64, y_dim = 64, z_dim = 64, ndim = 1, steps = 1, sleeptime = 0; + uint64_t numparticles, dims[1], offset_local[1], offset_remote[1], mysize[1]; + double t0, t1; + const char *obj_names[] = {"dX", "dY", "dZ", "Ux", "Uy", "Uz", "q", "i"}; + char obj_name[64]; + pdcid_t transfer_requests[8]; #ifdef ENABLE_MPI @@ -92,15 +89,17 @@ main(int argc, char **argv) dims[0] = numparticles * size; - x = (float *)malloc(numparticles * sizeof(float)); - y = (float *)malloc(numparticles * sizeof(float)); - z = (float *)malloc(numparticles * sizeof(float)); - px = (float *)malloc(numparticles * sizeof(float)); - py = (float *)malloc(numparticles * sizeof(float)); - pz = (float *)malloc(numparticles * sizeof(float)); + dx = (float *)malloc(numparticles * sizeof(float)); + dy = (float *)malloc(numparticles * sizeof(float)); + dz = (float *)malloc(numparticles * sizeof(float)); + ux = (float *)malloc(numparticles * sizeof(float)); + uy = (float *)malloc(numparticles * sizeof(float)); + uz = (float *)malloc(numparticles * sizeof(float)); + q = (float *)malloc(numparticles * sizeof(float)); + + id = (int *)malloc(numparticles * sizeof(int)); - id1 = (int *)malloc(numparticles * sizeof(int)); - id2 = (int *)malloc(numparticles * sizeof(int)); + void *data_ptrs[] = {&dx[0], &dy[0], &dz[0], &ux[0], &uy[0], &uz[0], &id[0], &q[0]}; // create a pdc pdc_id = PDCinit("pdc"); @@ -131,14 +130,14 @@ main(int argc, char **argv) PDCprop_set_obj_type(obj_prop_int, PDC_INT); for (uint64_t i = 0; i < numparticles; i++) { - id1[i] = i; - id2[i] = i * 2; - x[i] = uniform_random_number() * x_dim; - y[i] = uniform_random_number() * y_dim; - z[i] = ((float)id1[i] / numparticles) * z_dim; - px[i] = uniform_random_number() * x_dim; - py[i] = uniform_random_number() * y_dim; - pz[i] = ((float)id2[i] / numparticles) * z_dim; + id[i] = i; + q[i] = i * 2; + dx[i] = uniform_random_number() * x_dim; + dy[i] = uniform_random_number() * y_dim; + dz[i] = ((float)id[i] / numparticles) * z_dim; + ux[i] = uniform_random_number() * x_dim; + uy[i] = uniform_random_number() * y_dim; + uz[i] = (q[i] / numparticles) * z_dim; } offset_local[0] = 0; @@ -150,123 +149,50 @@ main(int argc, char **argv) region_remote = PDCregion_create(ndim, offset_remote, mysize); for (int iter = 0; iter < steps; iter++) { + // Change data for different steps for verification + id[0] = rank + iter; + q[0] = rank + iter * 2; + id[numparticles - 1] = rank - iter; + q[numparticles - 1] = rank - iter * 2; #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("\n[%s] #Step %d\n", cur_time, iter); + LOG_INFO("\n#Step %d\n", iter); t0 = MPI_Wtime(); #endif - PDCprop_set_obj_time_step(obj_prop_float, iter); - PDCprop_set_obj_time_step(obj_prop_int, iter); - - obj_xx = PDCobj_create_mpi(cont_id, "obj-var-xx", obj_prop_float, 0, comm); - if (obj_xx == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "x"); - return FAIL; - } - - obj_yy = PDCobj_create_mpi(cont_id, "obj-var-yy", obj_prop_float, 0, comm); - if (obj_yy == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "y"); - return FAIL; - } - obj_zz = PDCobj_create_mpi(cont_id, "obj-var-zz", obj_prop_float, 0, comm); - if (obj_zz == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "z"); - return FAIL; - } - obj_pxx = PDCobj_create_mpi(cont_id, "obj-var-pxx", obj_prop_float, 0, comm); - if (obj_pxx == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "px"); - return FAIL; - } - obj_pyy = PDCobj_create_mpi(cont_id, "obj-var-pyy", obj_prop_float, 0, comm); - if (obj_pyy == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "py"); - return FAIL; - } - obj_pzz = PDCobj_create_mpi(cont_id, "obj-var-pzz", obj_prop_float, 0, comm); - if (obj_pzz == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "pz"); - return FAIL; - } - obj_id11 = PDCobj_create_mpi(cont_id, "id11", obj_prop_int, 0, comm); - if (obj_id11 == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "id1"); - return FAIL; - } - obj_id22 = PDCobj_create_mpi(cont_id, "id22", obj_prop_int, 0, comm); - if (obj_id22 == 0) { - LOG_ERROR("Error getting an object id of %s from server\n", "id2"); - return FAIL; + for (int i = 0; i < 8; i++) { + sprintf(obj_name, "%s-%d", obj_names[i], iter); + pdcid_t obj_prop = (i < 7) ? obj_prop_float : obj_prop_int; + obj_ids[i] = PDCobj_create_mpi(cont_id, obj_name, obj_prop, 0, comm); + if (obj_ids[i] == 0) { + LOG_ERROR("Error getting an object id of %s from server\n", obj_name); + return FAIL; + } } #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Obj create time: %.5e\n", cur_time, t1 - t0); + LOG_INFO("Obj create time: %.5e\n", t1 - t0); #endif - transfer_requests[0] = - PDCregion_transfer_create(&x[0], PDC_WRITE, obj_xx, region_local, region_remote); - if (transfer_requests[0] == 0) { - LOG_ERROR("x transfer request creation failed\n"); - return FAIL; - } - transfer_requests[1] = - PDCregion_transfer_create(&y[0], PDC_WRITE, obj_yy, region_local, region_remote); - if (transfer_requests[1] == 0) { - LOG_ERROR("y transfer request creation failed\n"); - return FAIL; - } - transfer_requests[2] = - PDCregion_transfer_create(&z[0], PDC_WRITE, obj_zz, region_local, region_remote); - if (transfer_requests[2] == 0) { - LOG_ERROR("z transfer request creation failed\n"); - return FAIL; - } - transfer_requests[3] = - PDCregion_transfer_create(&px[0], PDC_WRITE, obj_pxx, region_local, region_remote); - if (transfer_requests[3] == 0) { - LOG_ERROR("px transfer request creation failed\n"); - return FAIL; - } - transfer_requests[4] = - PDCregion_transfer_create(&py[0], PDC_WRITE, obj_pyy, region_local, region_remote); - if (transfer_requests[4] == 0) { - LOG_ERROR("py transfer request creation failed\n"); - return FAIL; - } - transfer_requests[5] = - PDCregion_transfer_create(&pz[0], PDC_WRITE, obj_pzz, region_local, region_remote); - if (transfer_requests[5] == 0) { - LOG_ERROR("pz transfer request creation failed\n"); - return FAIL; - } - transfer_requests[6] = - PDCregion_transfer_create(&id1[0], PDC_WRITE, obj_id11, region_local, region_remote); - if (transfer_requests[6] == 0) { - LOG_ERROR("id1 transfer request creation failed\n"); - return FAIL; - } - transfer_requests[7] = - PDCregion_transfer_create(&id2[0], PDC_WRITE, obj_id22, region_local, region_remote); - if (transfer_requests[7] == 0) { - LOG_ERROR("id2 transfer request creation failed\n"); - return FAIL; + for (int i = 0; i < 8; i++) { + transfer_requests[i] = + PDCregion_transfer_create(data_ptrs[i], PDC_WRITE, obj_ids[i], region_local, region_remote); + if (transfer_requests[i] == 0) { + LOG_ERROR("%s transfer request creation failed\n", obj_names[i]); + return FAIL; + } } #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Transfer create time: %.5e\n", cur_time, t0 - t1); + LOG_INFO("Transfer create time: %.5e\n", t0 - t1); #endif #ifdef ENABLE_MPI @@ -281,19 +207,16 @@ main(int argc, char **argv) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Transfer start time: %.5e\n", cur_time, t1 - t0); + LOG_INFO("Transfer start time: %.5e\n", t1 - t0); #endif // Emulate compute with sleep if (iter != steps - 1) { - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Sleep start: %llu.00\n", cur_time, sleeptime); + LOG_INFO("Sleep start: %llu.00\n", sleeptime); sleep(sleeptime); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Sleep end: %llu.00\n", cur_time, sleeptime); + LOG_INFO("Sleep end: %llu.00\n", sleeptime); } #ifdef ENABLE_MPI @@ -309,9 +232,8 @@ main(int argc, char **argv) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Transfer wait time: %.5e\n", cur_time, t1 - t0); + LOG_INFO("Transfer wait time: %.5e\n", t1 - t0); #endif for (int j = 0; j < 8; j++) { @@ -324,50 +246,22 @@ main(int argc, char **argv) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Transfer close time: %.5e\n", cur_time, t0 - t1); + LOG_INFO("Transfer close time: %.5e\n", t0 - t1); #endif - if (PDCobj_close(obj_xx) != SUCCEED) { - LOG_ERROR("Failed to close obj_xx\n"); - return FAIL; - } - if (PDCobj_close(obj_yy) != SUCCEED) { - LOG_ERROR("Failed to close object obj_yy\n"); - return FAIL; - } - if (PDCobj_close(obj_zz) != SUCCEED) { - LOG_ERROR("Failed to close object obj_zz\n"); - return FAIL; - } - if (PDCobj_close(obj_pxx) != SUCCEED) { - LOG_ERROR("Failed to close object obj_pxx\n"); - return FAIL; - } - if (PDCobj_close(obj_pyy) != SUCCEED) { - LOG_ERROR("Failed to close object obj_pyy\n"); - return FAIL; - } - if (PDCobj_close(obj_pzz) != SUCCEED) { - LOG_ERROR("Failed to close object obj_pzz\n"); - return FAIL; - } - if (PDCobj_close(obj_id11) != SUCCEED) { - LOG_ERROR("Failed to close object obj_id11\n"); - return FAIL; - } - if (PDCobj_close(obj_id22) != SUCCEED) { - LOG_ERROR("Failed to close object obj_id22\n"); - return FAIL; + for (int i = 0; i < 8; i++) { + if (PDCobj_close(obj_ids[i]) != SUCCEED) { + LOG_ERROR("Failed to close object #%d\n", i); + return FAIL; + } } #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); - PDC_get_time_str(cur_time); if (rank == 0) - LOG_INFO("[%s] Obj close time: %.5e\n", cur_time, t1 - t0); + LOG_INFO("Obj close time: %.5e\n", t1 - t0); #endif } // End for steps @@ -401,14 +295,14 @@ main(int argc, char **argv) LOG_ERROR("Failed to close PDC\n"); return FAIL; } - free(x); - free(y); - free(z); - free(px); - free(py); - free(pz); - free(id1); - free(id2); + free(dx); + free(dy); + free(dz); + free(ux); + free(uy); + free(uz); + free(id); + free(q); #ifdef ENABLE_MPI MPI_Finalize(); #endif