...
 
Commits (2)
......@@ -5,10 +5,10 @@ else()
endif()
add_executable(mem main.cpp memory/HeapMemory.cpp ${CUDA_SOURCES} Memleak_check.cpp)
add_executable(mem main.cpp memory/HeapMemory.cpp ${CUDA_SOURCES} memory/ShmAllocator_manager.cpp memory/SemManager.cpp memory/ShmAllocator.cpp Memleak_check.cpp)
add_library(ofpmmemory STATIC memory/HeapMemory.cpp memory/PtrMemory.cpp Memleak_check.cpp ${CUDA_SOURCES})
add_library(ofpmmemory_se2 STATIC memory/HeapMemory.cpp memory/PtrMemory.cpp Memleak_check.cpp ${CUDA_SOURCES})
add_library(ofpmmemory STATIC memory/ShmAllocator_manager.cpp memory/SemManager.cpp memory/ShmAllocator.cpp memory/HeapMemory.cpp memory/PtrMemory.cpp Memleak_check.cpp ${CUDA_SOURCES})
add_library(ofpmmemory_se2 STATIC memory/ShmAllocator_manager.cpp memory/SemManager.cpp memory/ShmAllocator.cpp memory/HeapMemory.cpp memory/PtrMemory.cpp Memleak_check.cpp ${CUDA_SOURCES})
target_compile_definitions(ofpmmemory_se2 PUBLIC SE_CLASS2)
......@@ -54,6 +54,9 @@ install(FILES memory/ExtPreAlloc.hpp
memory/memory.hpp
memory/PtrMemory.hpp
memory/CudaMemory.cuh
memory/ShmAllocator_manager.hpp
memory/SemManager.hpp
memory/ShmAllocator.hpp
DESTINATION openfpm_devices/include/memory)
install(FILES util/print_stack.hpp
......
......@@ -98,13 +98,16 @@ public:
{}
//! Return the shared memory key
virtual key_t get_shmem_key()
virtual handle_shmem get_shmem_handle()
{
return -1;
handle_shmem sh;
sh.id = -1;
return sh;
}
//! Return the shared memory key
virtual void set_shmem_key(key_t k)
virtual void set_shmem_handle(handle_shmem sh)
{}
//! flush the memory
......
......@@ -21,15 +21,13 @@ static const int extra_pad = 512;
byte * HeapMemory::allocate_mem(size_t size)
{
if (key == -1)
if (sh_handle.id == -1)
{return new byte[size];}
else
{
shmid = shmget(key,size,0666|IPC_CREAT);
std::cout << "Allocating using shared memory" << std::endl;
std::cout << errno << std::endl;
return (byte *)shmat(shmid,(void *)0,0);
return (byte *)create_shmanager().alloc(sh_handle, size);
}
}
......@@ -96,10 +94,12 @@ void HeapMemory::destroy()
if (dmOrig != NULL)
{
if (key == -1)
if (sh_handle.id == -1)
{delete [] dmOrig;}
else
{/*shmctl(shmid, IPC_RMID, NULL);*/}
{
create_shmanager().free(sh_handle, dmOrig);
}
}
sz = 0;
......@@ -284,7 +284,7 @@ const void * HeapMemory::getPointer() const
return dm;
}
void HeapMemory::set_memory_name(const char * pathname, int proj_id)
{
key = ftok(pathname,proj_id);
}
//void HeapMemory::set_memory_name(handle_shmem shid)
//{
// sh_handle = shid;
//}
......@@ -13,6 +13,7 @@
#include <cstddef>
#include <cstdint>
#include <iostream>
#include "ShmAllocator_manager.hpp"
typedef unsigned char byte;
......@@ -52,7 +53,7 @@ class HeapMemory : public memory
long int ref_cnt;
//! key for shared memory
key_t key = -1;
handle_shmem sh_handle;
//! shared memory id
int shmid = -1;
......@@ -92,9 +93,6 @@ public:
//! get a device pointer for HeapMemory getPointer and getDevicePointer are equivalents
virtual void * getDevicePointer();
//! \see memory.hpp
virtual void set_memory_name(const char * pathname, int proj_id);
/*! \brief fill host and device memory with the selected byte
*
*
......@@ -128,15 +126,15 @@ public:
}
//! Return the shared memory key
virtual key_t get_shmem_key()
virtual handle_shmem get_shmem_handle()
{
return key;
return sh_handle;
}
//! Return the shared memory key
virtual void set_shmem_key(key_t k)
virtual void set_shmem_handle(handle_shmem sh)
{
key = k;
sh_handle = sh;
}
/*! \brief Allocated Memory is never initialized
......@@ -160,12 +158,14 @@ public:
HeapMemory(const HeapMemory & mem)
:HeapMemory()
{
sh_handle.id = -1;
allocate(mem.size());
copy(mem);
}
HeapMemory(HeapMemory && mem) noexcept
{
sh_handle.id = -1;
//! move
alignement = mem.alignement;
sz = mem.sz;
......@@ -181,7 +181,8 @@ public:
}
//! Constructor, we choose a default alignment of 32 for avx
HeapMemory():alignement(MEM_ALIGNMENT),sz(0),dm(NULL),dmOrig(NULL),ref_cnt(0) {};
HeapMemory():alignement(MEM_ALIGNMENT),sz(0),dm(NULL),dmOrig(NULL),ref_cnt(0)
{sh_handle.id = -1;};
virtual ~HeapMemory() noexcept
{
......
......@@ -118,14 +118,17 @@ public:
virtual void set_memory_name(const char * pathname, int proj_id)
{}
//! Return the shared memory key
virtual key_t get_shmem_key()
//! Return the shared memory handle
virtual handle_shmem get_shmem_handle()
{
return -1;
handle_shmem sh;
sh.id = -1;
return sh;
}
//! Return the shared memory key
virtual void set_shmem_key(key_t k)
//! Return the shared memory handle
virtual void set_shmem_handle(handle_shmem sh)
{}
/*! \brief Return true if the device and the host pointer are the same
......
/*
* Semaphore manager for both producer and consumer
*
*
*/
#include <cstdlib>
#include "SemManager.hpp"
#define PROJ_ID(rank, toggle) (2*(rank)+1+(toggle)) // generate proj_id to send to ftok (later should be more complex)
#define TESTPRINT if (verbose) printf
SemManager::SemManager(std::string pname, int rank, bool verbose, bool ismain) : pname(pname), rank(rank), verbose(verbose), ismain(ismain)
{
for (int i = 0; i < NKEYS; ++i) {
TESTPRINT("rank:%d\ttoggle:%d\tid:%d\t", rank, i, PROJ_ID(rank, i)); // test
keys[i] = ftok(pname.data(), PROJ_ID(rank, i));
TESTPRINT("key:%d\n", keys[i]); // test
// initialize semaphore
semids[i] = semget(keys[i], NSEMS, 0666|IPC_CREAT);
}
}
SemManager::~SemManager()
{
if (ismain) {
for (int i = 0; i < NKEYS; ++i) {
// delete semaphore
TESTPRINT("deleting semaphore %d\n", i);
semctl(semids[i], 0, IPC_RMID);
TESTPRINT("deleted semaphore %d\n", i);
}
}
TESTPRINT("deleted SemManager\n");
}
const int &SemManager::operator[](int keyNo)
{
return keys[keyNo];
}
void SemManager::set(int keyNo, int semNo, int value)
{
sem_attr.val = value;
semctl(semids[keyNo], semNo, SETVAL, sem_attr);
}
int SemManager::get(int keyNo, int semNo)
{
return semctl(semids[keyNo], semNo, GETVAL);
}
void SemManager::incr(int keyNo, int semNo)
{
semops[0].sem_num = semNo;
semops[0].sem_op = 1;
semops[0].sem_flg = 0;
if (semop(semids[keyNo], semops, 1) == -1) {
perror("semop"); std::exit(1);
}
TESTPRINT("incremented semaphore %d of key %d\n", semNo, keyNo); // test
}
void SemManager::decr(int keyNo, int semNo)
{
semops[0].sem_num = semNo;
semops[0].sem_op = -1;
semops[0].sem_flg = 0;
if (semop(semids[keyNo], semops, 1) == -1) {
perror("semop"); std::exit(1);
}
TESTPRINT("decremented semaphore %d of key %d\n", semNo, keyNo); // test
}
void SemManager::wait(int keyNo, int semNo, int value)
{
TESTPRINT("waiting for semaphore %d of key %d to reach %d\n", semNo, keyNo, value); // test
if (value == 0) {
semops[0].sem_num = semNo;
semops[0].sem_op = 0;
semops[0].sem_flg = 0;
if (semop(semids[keyNo], semops, 1) == -1) {
perror("semop"); std::exit(1);
}
} else {
// decrement by value, wait for zero (necessary if semaphore was initially higher than value, which in our case doesn't happen), then increment by value
semops[0].sem_num = semNo;
semops[0].sem_op = -value;
semops[0].sem_flg = 0;
semops[1].sem_num = semNo;
semops[1].sem_op = 0;
semops[1].sem_flg = 0;
semops[2].sem_num = semNo;
semops[2].sem_op = value;
semops[2].sem_flg = 0;
if (semop(semids[keyNo], semops, 3) == -1) {
perror("semop"); std::exit(1);
}
}
TESTPRINT("waited for semaphore %d of key %d\n", semNo, keyNo); // test
}
void SemManager::waitgeq(int keyNo, int semNo, int value)
{
if (value == 0)
return;
TESTPRINT("waiting for semaphore %d of key %d to reach at least %d\n", semNo, keyNo, value); // test
// decrement by value, wait for zero (necessary if semaphore was initially higher than value, which in our case doesn't happen), then increment by value
semops[0].sem_num = semNo;
semops[0].sem_op = -value;
semops[0].sem_flg = 0;
semops[1].sem_num = semNo;
semops[1].sem_op = value;
semops[1].sem_flg = 0;
if (semop(semids[keyNo], semops, 2) == -1) {
perror("semop"); std::exit(1);
}
TESTPRINT("waited for semaphore %d of key %d\n", semNo, keyNo); // test
}
\ No newline at end of file
/*
* Semaphore manager for both producer and consumer
*
*
*/
#ifndef SEM_MANAGER_HPP
#define SEM_MANAGER_HPP
#include <string>
#include <sys/sem.h>
#ifndef __APPLE__
union semun {
int val; /* Value for SETVAL */
struct semid_ds *buf; /* Buffer for IPC_STAT, IPC_SET */
unsigned short *array; /* Array for GETALL, SETALL */
struct seminfo *__buf; /* Buffer for IPC_INFO
(Linux-specific) */
};
#endif
#define NKEYS 2 // number of keys per each rank
#define NSEMS 2 // number of semaphores per key (one for consumer, one for producer)
#define SEMOPS 10 // max number of consecutive semaphore calls supported
class SemManager {
std::string pname;
int rank;
bool ismain; // whether to manage and delete semaphores, true by default
bool verbose;
int keys[NKEYS]; // keys to be used and toggled
int semids[NKEYS]; // the semaphore id used for each key
// for semaphore calls
union semun sem_attr;
struct sembuf semops[SEMOPS];
public:
SemManager(std::string pname, int rank, bool verbose = true, bool ismain = true);
~SemManager();
const int &operator[](int keyNo); // return key[keyNo]
void set(int keyNo, int semNo, int value); // directly set semaphore value
int get(int keyNo, int semNo); // directly get semaphore value
void incr(int keyNo, int semNo); // increment semaphore value
void decr(int keyNo, int semNo); // decrement semaphore value, wait if semaphore equal to 0
void wait(int keyNo, int semNo, int value = 0); // wait until semaphore equal to value (blocking)
void waitgeq(int keyNo, int semNo, int value); // wait until semaphore at least value
};
#endif
\ No newline at end of file
/*
* Shared memory allocator class
*
*
*
*/
#include <iostream>
#include <future>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <cstdlib>
#include "ShmAllocator.hpp"
#define CONSEM 0 // index of semaphore for consumer
#define PROSEM 1 // index of semaphore for producer
#define KEYINIT -2 // initial value of current_key to signify no previous memory allocated
#define PROJ_ID(rank, toggle) (2*(rank)+1+(toggle)) // generate proj_id to send to ftok
#define TESTPRINT if (verbose) printf
ShmAllocator::ShmAllocator(std::string pname, int rank, bool verbose) : sems(pname, rank, verbose, true), current_key(KEYINIT), verbose(verbose)
{
for (int i = 0; i < NKEYS; ++i) {
shmids[i] = -1;
ptrs[i] = NULL;
// used[i] = false;
// currently, no consumers or producers
sems.set(i, CONSEM, 0); // no consumers -> 0
sems.set(i, PROSEM, 0); // no producers -> 0
}
}
ShmAllocator::~ShmAllocator()
{
for (int i = 0; i < NKEYS; ++i) {
// delete pointer if it is used
sems.set(i, CONSEM, 0); // don't wait for consumer to finish
TESTPRINT("set semaphore %d\n", i);
shm_free(ptrs[i]); // TODO this call seems to freeze sometimes
TESTPRINT("freed pointer %d\n", i);
// or call out.wait_for() for a given timeout duration, making out a class field and not a static variable for shm_free
}
TESTPRINT("deleted ShmAllocator\n");
}
void *ShmAllocator::shm_alloc(size_t size)
{
current_key &= 1; // -2 becomes 0
TESTPRINT("trying lock for %d\n", current_key);
if (!used[current_key].try_lock()) { // if current key locked, toggle it
current_key ^= 1;
TESTPRINT("trying lock for %d\n", current_key);
if (!used[current_key].try_lock()) // try locking other key
return malloc(size); // allocate from heap memory if both keys used
}
TESTPRINT("locked %d\n", current_key);
TESTPRINT("rank:%d\tkey:%d\n", current_key, sems[current_key]); // test
// assert !used[current_key]; user must not be able to allocate more than twice
// wait for current key to stop being used by consumer
// technically used is also like a semaphore
// allocate memory with new key
if ((shmids[current_key] = shmget(sems[current_key], size, 0666|IPC_CREAT)) == -1) {
perror("shmget"); std::exit(1);
}
TESTPRINT("shmid:%d\n", shmids[current_key]); // test
if ((long) (ptrs[current_key] = shmat(shmids[current_key], NULL, 0)) == -1) {
perror("shmat"); std::exit(1);
}
TESTPRINT("ptr:%ld\n", (long) ptrs[current_key]); // test
// mark new key as used
// used[current_key] = true;
// increment semaphore for new key to signal consumer
sems.incr(current_key, PROSEM);
// return pointer
return ptrs[current_key];
}
void ShmAllocator::shm_free(void *ptr)
{
static std::future<void> out;
// return if null pointer
if (ptr == NULL)
return;
// find the key that ptr refers to
int key;
for (key = 0; key < NKEYS; ++key)
if (ptrs[key] == ptr) // here, actually check if ptr lies in the interval allocated for ptrs[key], possibly storing sizes
break;
// if no key found, return
if (key == NKEYS) {
TESTPRINT("Pointer %ld not found in shm\n", (long) ptr);
free(ptr);
return;
}
TESTPRINT("Pointer %ld found assigned to key %d\n", (long) ptr, key);
// assert used[key]
// mark key as unused
// used[key] = false;
// shmdt(ptr); // better here than after waiting; pointer should be unusable after free is called
// decrement semaphore for key
if (sems.get(key, PROSEM) != 0)
sems.decr(key, PROSEM);
// wait_del(key);
out = std::async(std::launch::async, &ShmAllocator::wait_del, this, key);
}
void ShmAllocator::wait_del(int key)
{
// wait for consumer to stop using key
sems.wait(key, CONSEM, 0); // need to check if this is busy waiting
// execute after waiting so that another allocate call to the key would not mess things up
// used[key] = false; // ensure that ptrs[key] and shmids[key] only change when used[key] is false
// deallocate shared memory
shmdt(ptrs[key]); // ptrs[key] must not have changed
shmctl(shmids[key], IPC_RMID, NULL); // shmid[key] must not have changed
ptrs[key] = NULL;
shmids[key] = -1;
// mark key as unused by consumer
used[key].unlock();
}
\ No newline at end of file
/*
* Shared memory allocator class
*
* Provides two public functions like malloc and free
*
* shm_alloc(size): given size in bytes, allocate shared memory of given size, return pointer
* - toggle current key
* - allocate shared memory with new key
* - mark new key as used, both in used[] and in the semaphore
*
* shm_free(ptr): given pointer, remove shared memory; do nothing if ptr is NULL
* - find key associated to pointer
* - increment semaphore
* - asynchronously (calling wait_del):
* - wait for consumer to release key
* - mark key as unused
* - delete shared memory
*/
#ifndef SHM_ALLOC_HPP_
#define SHM_ALLOC_HPP_
#include <string>
#include <mutex>
#include "SemManager.hpp"
#define SHMINIT -1 // value of shmids[key] when no shared memory is associated to key
class ShmAllocator {
SemManager sems;
bool verbose;
// bool used[NKEYS]; // whether each key is currently used (allocated and not yet deleted, incl. not released by consumer)
std::mutex used[NKEYS]; // lock when currently used; only after deallocation unlocks mutex can memory be allocated again
int shmids[NKEYS]; // the shared memory id used for each key (-1 if not used)
void *ptrs[NKEYS]; // pointers allocated for each key (NULL if not allocated)
int current_key; // takes values 0 or 1; most recent memory allocated using keys[current_key]
void wait_del(int key); // wait to delete ptrs[key], called from shm_free
public:
ShmAllocator(std::string pname, int rank, bool verbose = false); // generate two keys per rank, pass pname to ftok, initialize semaphores
~ShmAllocator(); // delete semaphores and any remaining memory segments
void *shm_alloc(size_t size); // allocate shared memory of given size
void shm_free(void *ptr); // free shared memory segment associated to pointer, which may be NULL
};
#endif
\ No newline at end of file
......@@ -20,6 +20,7 @@ typedef long int mem_id;
#include <stddef.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include "ShmAllocator_manager.hpp"
class memory
{
......@@ -162,24 +163,17 @@ class memory
*/
virtual void fill(unsigned char c) = 0;
/*! \brief Set the name of the memory
*
* \note in case of HeapMemory is the name of the file to generate a key for the shared memory
*
*/
virtual void set_memory_name(const char * pathname, int proj_id) = 0;
/*! \brief
*
*
*/
virtual key_t get_shmem_key() = 0;
virtual handle_shmem get_shmem_handle() = 0;
/*! \brief
*
*
*/
virtual void set_shmem_key(key_t key) = 0;
virtual void set_shmem_handle(handle_shmem) = 0;
};
#endif