LCOV - code coverage report
Current view: top level - src/plugins/mmapstorage - mmapstorage.c (source / functions) Hit Total Coverage
Test: coverage-filtered.info Lines: 383 386 99.2 %
Date: 2019-09-12 12:28:41 Functions: 24 26 92.3 %

          Line data    Source code
       1             : /**
       2             :  * @file
       3             :  *
       4             :  * @brief Source for mmapstorage plugin
       5             :  *
       6             :  * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
       7             :  *
       8             :  */
       9             : #define _XOPEN_SOURCE 600
      10             : 
      11             : /* -- Imports --------------------------------------------------------------------------------------------------------------------------- */
      12             : 
      13             : #include "mmapstorage.h"
      14             : #include "dynarray.h"
      15             : #include "internal.h"
      16             : 
      17             : #include <kdbassert.h>
      18             : #include <kdberrors.h>
      19             : #include <kdbhelper.h>
      20             : #include <kdblogger.h>
      21             : #include <kdbprivate.h>
      22             : 
      23             : #ifdef HAVE_KDBCONFIG_H
      24             : #include "kdbconfig.h"
      25             : #endif
      26             : 
      27             : #include <errno.h>
      28             : #include <fcntl.h>     // fcntl()
      29             : #include <limits.h>    // SSIZE_MAX
      30             : #include <stdint.h>    // uintN_t, uintptr_t
      31             : #include <stdio.h>     // fopen(), fileno()
      32             : #include <stdlib.h>    // strtol()
      33             : #include <string.h>    // memcmp()
      34             : #include <sys/mman.h>  // mmap()
      35             : #include <sys/stat.h>  // stat(), fstat()
      36             : #include <sys/types.h> // ftruncate (), size_t
      37             : #include <unistd.h>    // close(), ftruncate(), unlink(), read(), write()
      38             : 
      39             : #ifdef ELEKTRA_MMAP_CHECKSUM
      40             : #include <zlib.h> // crc32()
      41             : #endif
      42             : 
      43             : /* -- Global declarations---------------------------------------------------------------------------------------------------------------- */
      44             : 
      45             : static KeySet magicKeySet;
      46             : static Key magicKey;
      47             : static MmapMetaData magicMmapMetaData;
      48             : 
      49             : typedef enum
      50             : {
      51             :         MODE_STORAGE = 1,
      52             :         MODE_GLOBALCACHE = 1 << 1,
      53             :         MODE_NONREGULAR_FILE = 1 << 2
      54             : } PluginMode;
      55             : 
      56             : /* -- File handling --------------------------------------------------------------------------------------------------------------------- */
      57             : 
      58             : /**
      59             :  * @brief Wrapper for open().
      60             :  *
      61             :  * @param parentKey containing the filename
      62             :  * @param flags file access mode
      63             :  * @param openMode file mode bits when file is created
      64             :  * @param mode the current plugin mode
      65             :  *
      66             :  * @return file descriptor
      67             :  */
      68             : static int openFile (Key * parentKey, int flag, mode_t openMode, PluginMode mode)
      69             : {
      70             :         int fd;
      71             :         ELEKTRA_LOG_DEBUG ("opening file %s", keyString (parentKey));
      72             : 
      73         577 :         if ((fd = open (keyString (parentKey), flag, openMode)) == -1)
      74             :         {
      75             :                 ELEKTRA_MMAP_LOG_WARNING ("error opening file %s", keyString (parentKey));
      76             :         }
      77             :         return fd;
      78             : }
      79             : 
      80             : /**
      81             :  * @brief Wrapper for ftruncate().
      82             :  *
      83             :  * @param fd the file descriptor
      84             :  * @param mmapsize size of the mapped region
      85             :  * @param parentKey holding the filename, for debug purposes
      86             :  * @param mode the current plugin mode
      87             :  *
      88             :  * @retval 1 on success
      89             :  * @retval -1 if ftruncate() failed
      90             :  */
      91             : static int truncateFile (int fd, size_t mmapsize, Key * parentKey ELEKTRA_UNUSED, PluginMode mode)
      92             : {
      93             :         ELEKTRA_LOG_DEBUG ("truncating file %s", keyString (parentKey));
      94             : 
      95         342 :         if ((ftruncate (fd, mmapsize)) == -1)
      96             :         {
      97             :                 ELEKTRA_MMAP_LOG_WARNING ("error truncating file %s", keyString (parentKey));
      98             :                 return -1;
      99             :         }
     100             :         return 1;
     101             : }
     102             : 
     103             : /**
     104             :  * @brief Wrapper for fstat().
     105             :  *
     106             :  * @param fd the file descriptor
     107             :  * @param sbuf the stat structure
     108             :  * @param parentKey holding the filename
     109             :  * @param mode the current plugin mode
     110             :  *
     111             :  * @retval 1 on success
     112             :  * @retval -1 if fstat() failed
     113             :  */
     114         581 : static int fstatFile (int fd, struct stat * sbuf, Key * parentKey ELEKTRA_UNUSED, PluginMode mode)
     115             : {
     116             :         ELEKTRA_LOG_DEBUG ("fstat() on file %s", keyString (parentKey));
     117             : 
     118         581 :         memset (sbuf, 0, sizeof (struct stat));
     119         581 :         if (fstat (fd, sbuf) == -1)
     120             :         {
     121             :                 ELEKTRA_MMAP_LOG_WARNING ("error on fstat() for file %s", keyString (parentKey));
     122             :                 return -1;
     123             :         }
     124             :         return 1;
     125             : }
     126             : 
     127             : /**
     128             :  * @brief Wrapper for mmap().
     129             :  *
     130             :  * @param addr address hint, where the mapping should start
     131             :  * @param fd file descriptor of the file to be mapped
     132             :  * @param mmapSize size of the mapped region
     133             :  * @param mapOpts mmap flags (MAP_PRIVATE, MAP_FIXED, ...)
     134             :  * @param parentKey holding the filename, for debug purposes
     135             :  * @param mode the current plugin mode
     136             :  *
     137             :  * @return pointer to mapped region on success, MAP_FAILED on failure
     138             :  */
     139         577 : static char * mmapFile (void * addr, int fd, size_t mmapSize, int mapOpts, Key * parentKey ELEKTRA_UNUSED, PluginMode mode)
     140             : {
     141             :         ELEKTRA_LOG_DEBUG ("mapping file %s", keyString (parentKey));
     142             : 
     143         577 :         char * mappedRegion = MAP_FAILED;
     144         577 :         if (test_bit (mode, MODE_NONREGULAR_FILE))
     145             :         {
     146           2 :                 mappedRegion = elektraCalloc (mmapSize);
     147           2 :                 if (!mappedRegion)
     148             :                 {
     149             :                         ELEKTRA_MMAP_LOG_WARNING ("error allocating buffer for file %s\nmmapSize: %zu", keyString (parentKey), mmapSize);
     150             :                         return MAP_FAILED;
     151             :                 }
     152             :         }
     153             :         else
     154             :         {
     155         575 :                 mappedRegion = mmap (addr, mmapSize, PROT_READ | PROT_WRITE, mapOpts, fd, 0);
     156         575 :                 if (mappedRegion == MAP_FAILED)
     157             :                 {
     158             :                         ELEKTRA_MMAP_LOG_WARNING ("error mapping file %s\nmmapSize: %zu", keyString (parentKey), mmapSize);
     159             :                         return MAP_FAILED;
     160             :                 }
     161             :         }
     162             : 
     163             :         return mappedRegion;
     164             : }
     165             : 
     166             : 
     167             : /**
     168             :  * @brief Copy file
     169             :  *
     170             :  * @param sourceFd source file descriptor
     171             :  * @param destFd destination file descriptor
     172             :  *
     173             :  * @retval 0 on success
     174             :  * @retval -1 if any error occured
     175             :  */
     176           2 : static int copyFile (int sourceFd, int destFd)
     177             : {
     178           2 :         ssize_t readBytes = 0;
     179             :         char buf[ELEKTRA_MMAP_BUFSIZE];
     180             : 
     181           6 :         while ((readBytes = read (sourceFd, buf, ELEKTRA_MMAP_BUFSIZE)) > 0)
     182             :         {
     183             :                 if (readBytes <= 0)
     184             :                 {
     185             :                         if (errno == EINTR)
     186             :                                 continue;
     187             :                         else
     188             :                                 return -1;
     189             :                 }
     190             : 
     191             :                 char * pos = buf;
     192             :                 ssize_t writtenBytes = 0;
     193           4 :                 while (readBytes > 0)
     194             :                 {
     195           2 :                         writtenBytes = write (destFd, buf, readBytes);
     196           2 :                         if (writtenBytes <= 0)
     197             :                         {
     198           0 :                                 if (errno == EINTR)
     199           0 :                                         continue;
     200             :                                 else
     201             :                                         return -1;
     202             :                         }
     203           2 :                         readBytes -= writtenBytes;
     204           2 :                         pos += writtenBytes;
     205             :                 }
     206             :         }
     207             : 
     208             :         return 0;
     209             : }
     210             : 
     211             : /**
     212             :  * @brief Copy file to anonymous temporary file
     213             :  *
     214             :  * This function is needed to make mmapstorage compatible
     215             :  * with any file. The `mmap()` syscall only supports regular
     216             :  * files.
     217             :  *
     218             :  * @param fd source file descriptor
     219             :  * @param sbuf the stat structure
     220             :  * @param parentKey holding the file name
     221             :  * @param mode the current plugin mode
     222             :  *
     223             :  * @retval 0 on success
     224             :  * @retval -1 if any error occured
     225             :  */
     226           2 : static int copyToAnonymousTempfile (int fd, struct stat * sbuf, Key * parentKey, PluginMode mode)
     227             : {
     228           2 :         int tmpFd = 0;
     229           2 :         char name[] = ELEKTRA_MMAP_TMP_NAME;
     230           2 :         if ((tmpFd = mkstemp (name)) < 0)
     231             :         {
     232             :                 return -1;
     233             :         }
     234             : 
     235             :         ELEKTRA_LOG_DEBUG ("using anon tmp file: %s", name);
     236           2 :         keySetString (parentKey, name);
     237             :         // use anonymous file
     238           2 :         if (unlink (name) != 0)
     239             :         {
     240             :                 ELEKTRA_MMAP_LOG_WARNING ("could not unlink");
     241             :                 return -1;
     242             :         }
     243             : 
     244           2 :         copyFile (fd, tmpFd);
     245             : 
     246           2 :         if (close (fd) != 0)
     247             :         {
     248             :                 ELEKTRA_MMAP_LOG_WARNING ("could not close");
     249             :                 return -1;
     250             :         }
     251             : 
     252             :         // replace old file
     253           2 :         fd = tmpFd;
     254           2 :         if (fstatFile (fd, sbuf, parentKey, mode) != 1)
     255             :         {
     256             :                 return -1;
     257             :         }
     258             : 
     259             :         return fd;
     260             : }
     261             : 
     262             : /* -- Internal Functions  --------------------------------------------------------------------------------------------------------------- */
     263             : 
     264             : /**
     265             :  * @brief MmapHeader initializer.
     266             :  *
     267             :  * @param mmapHeader to initialize
     268             :  */
     269             : static void initHeader (MmapHeader * mmapHeader)
     270             : {
     271         344 :         memset (mmapHeader, 0, SIZEOF_MMAPHEADER);
     272         344 :         mmapHeader->mmapMagicNumber = ELEKTRA_MAGIC_MMAP_NUMBER;
     273         344 :         mmapHeader->formatVersion = ELEKTRA_MMAP_FORMAT_VERSION;
     274             : #ifdef ELEKTRA_MMAP_CHECKSUM
     275         132 :         mmapHeader->formatFlags = MMAP_FLAG_CHECKSUM;
     276             : #endif
     277             : }
     278             : 
     279             : /**
     280             :  * @brief MmapMetaData initializer.
     281             :  *
     282             :  * @param mmapMetaData to initialize
     283             :  */
     284             : static void initMetaData (MmapMetaData * mmapMetaData)
     285             : {
     286         344 :         memset (mmapMetaData, 0, SIZEOF_MMAPMETADATA);
     287             : }
     288             : 
     289             : /**
     290             :  * @brief MmapFooter initializer.
     291             :  *
     292             :  * @param mmapFooter to initialize
     293             :  */
     294             : static void initFooter (MmapFooter * mmapFooter)
     295             : {
     296             :         memset (mmapFooter, 0, SIZEOF_MMAPFOOTER);
     297         344 :         mmapFooter->mmapMagicNumber = ELEKTRA_MAGIC_MMAP_NUMBER;
     298             : }
     299             : 
     300             : /**
     301             :  * @brief Reads the MmapHeader and MmapMetaData from a file.
     302             :  *
     303             :  * @param fp file pointer to read the data from
     304             :  * @param mmapHeader buffer where the MmapHeader is stored
     305             :  * @param mmapMetaData buffer where the MmapMetaData is stored
     306             :  *
     307             :  * @retval -1 if magic number or format version was wrong
     308             :  * @retval 0 if read succeeded, the magic number was read correctly and the format version matched
     309             :  */
     310             : static int readHeader (const char * mappedRegion, MmapHeader ** mmapHeader, MmapMetaData ** mmapMetaData)
     311             : {
     312         233 :         *mmapHeader = (MmapHeader *) mappedRegion;
     313         233 :         *mmapMetaData = (MmapMetaData *) (mappedRegion + OFFSET_MMAPMETADATA);
     314             : 
     315         233 :         if ((*mmapHeader)->mmapMagicNumber == ELEKTRA_MAGIC_MMAP_NUMBER && (*mmapHeader)->formatVersion == ELEKTRA_MMAP_FORMAT_VERSION)
     316             :         {
     317             :                 return 0;
     318             :         }
     319             : 
     320             :         return -1;
     321             : }
     322             : 
     323             : /**
     324             :  * @brief Reads the MmapFooter from the end of the mapped region.
     325             :  *
     326             :  * @param mappedRegion pointer to the mapped region
     327             :  * @param mmapHeader the MmapHeader containing the size of the allocation
     328             :  *
     329             :  * @retval -1 if the magic number did not match
     330             :  * @retval 0 if the magic number matched
     331             :  */
     332             : static int readFooter (const char * mappedRegion, MmapHeader * mmapHeader)
     333             : {
     334         227 :         MmapFooter * mmapFooter = (MmapFooter *) (mappedRegion + mmapHeader->allocSize - SIZEOF_MMAPFOOTER);
     335             : 
     336         227 :         if (mmapFooter->mmapMagicNumber == ELEKTRA_MAGIC_MMAP_NUMBER)
     337             :         {
     338             :                 return 0;
     339             :         }
     340             : 
     341             :         return -1;
     342             : }
     343             : 
     344             : /**
     345             :  * @brief Writes the magic data for consistency checks.
     346             :  *
     347             :  * @param dest to write the data to.
     348             :  */
     349         344 : static void writeMagicData (const char * mappedRegion)
     350             : {
     351         344 :         KeySet * destKeySet = (KeySet *) (mappedRegion + OFFSET_MAGIC_KEYSET);
     352         344 :         Key * keyPtr = (Key *) (mappedRegion + OFFSET_MAGIC_KEY);
     353         344 :         MmapMetaData * mmapMetaData = (MmapMetaData *) (mappedRegion + OFFSET_MAGIC_MMAPMETADATA);
     354         344 :         memcpy (destKeySet, &magicKeySet, SIZEOF_KEYSET);
     355         344 :         memcpy (keyPtr, &magicKey, SIZEOF_KEY);
     356         344 :         memcpy (mmapMetaData, &magicMmapMetaData, SIZEOF_MMAPMETADATA);
     357         344 : }
     358             : 
     359             : /* -- Verification Functions  ----------------------------------------------------------------------------------------------------------- */
     360             : 
     361             : /**
     362             :  * @brief Generate magic number depending on pointer size.
     363             :  *
     364             :  * Generates magic number to detect arbitrary byte swaps.
     365             :  * For each byte of the number, a different value is assigned.
     366             :  * E.g.: Systems with 8-byte pointers will have the magic number 0x0706050403020100.
     367             :  *
     368             :  * @return generated magic number
     369             :  */
     370             : static uintptr_t generateMagicNumber (void)
     371             : {
     372         608 :         uintptr_t ret = 0;
     373             : 
     374         608 :         size_t ptrBytes = sizeof (void *);
     375        5472 :         for (uintptr_t i = 0; i < ptrBytes; ++i)
     376             :         {
     377        4864 :                 ret |= (i << (i * 8));
     378             :         }
     379             : 
     380             :         return ret;
     381             : }
     382             : 
     383             : /**
     384             :  * @brief Magic KeySet initializer.
     385             :  *
     386             :  * @param magicNumber to detect arbitrary byte-swaps
     387             :  */
     388             : static void initMagicKeySet (const uintptr_t magicNumber)
     389             : {
     390          96 :         magicKeySet.array = (Key **) magicNumber;
     391          96 :         magicKeySet.size = SIZE_MAX;
     392          96 :         magicKeySet.alloc = 0;
     393          96 :         magicKeySet.cursor = (Key *) ~magicNumber;
     394          96 :         magicKeySet.current = SIZE_MAX / 2;
     395          96 :         magicKeySet.flags = KS_FLAG_MMAP_ARRAY | KS_FLAG_SYNC;
     396             : #ifdef ELEKTRA_ENABLE_OPTIMIZATIONS
     397          96 :         magicKeySet.opmphm = (Opmphm *) ELEKTRA_MMAP_MAGIC_BOM;
     398          96 :         magicKeySet.opmphmPredictor = 0;
     399             : #endif
     400             : }
     401             : 
     402             : /**
     403             :  * @brief Magic Key initializer.
     404             :  *
     405             :  * @param magicNumber to detect arbitrary byte-swaps
     406             :  */
     407             : static void initMagicKey (const uintptr_t magicNumber)
     408             : {
     409          96 :         magicKey.data.v = (void *) ~magicNumber;
     410          96 :         magicKey.dataSize = SIZE_MAX;
     411          96 :         magicKey.key = (char *) magicNumber;
     412          96 :         magicKey.keySize = UINT16_MAX;
     413          96 :         magicKey.keyUSize = 0;
     414          96 :         magicKey.flags = KEY_FLAG_MMAP_STRUCT | KEY_FLAG_MMAP_DATA | KEY_FLAG_MMAP_KEY | KEY_FLAG_SYNC;
     415          96 :         magicKey.ksReference = SIZE_MAX / 2;
     416          96 :         magicKey.meta = (KeySet *) ELEKTRA_MMAP_MAGIC_BOM;
     417             : }
     418             : 
     419             : /**
     420             :  * @brief Magic MmapMetaData initializer.
     421             :  *
     422             :  * @param magicNumber to detect arbitrary byte-swaps
     423             :  */
     424             : static void initMagicMmapMetaData (void)
     425             : {
     426          96 :         magicMmapMetaData.numKeySets = SIZE_MAX;
     427          96 :         magicMmapMetaData.ksAlloc = 0;
     428          96 :         magicMmapMetaData.numKeys = SIZE_MAX / 2;
     429             : }
     430             : 
     431             : /**
     432             :  * @brief Verify the magic KeySet.
     433             :  *
     434             :  * @param ks keyset to verify
     435             :  *
     436             :  * @retval 0 if magic KeySet is consistent
     437             :  * @retval -1 if magic KeySet is inconsistent
     438             :  */
     439             : static int verifyMagicKeySet (KeySet * ks)
     440             : {
     441         225 :         if (!ks) return -1;
     442         225 :         return memcmp (ks, &magicKeySet, SIZEOF_KEYSET);
     443             : }
     444             : 
     445             : /**
     446             :  * @brief Verify the magic Key.
     447             :  *
     448             :  * @param key to verify
     449             :  *
     450             :  * @retval 0 if magic Key is consistent
     451             :  * @retval -1 if magic Key is inconsistent
     452             :  */
     453             : static int verifyMagicKey (Key * key)
     454             : {
     455         225 :         if (!key) return -1;
     456         225 :         return memcmp (key, &magicKey, SIZEOF_KEY);
     457             : }
     458             : 
     459             : /**
     460             :  * @brief Verify the magic MmapMetaData.
     461             :  *
     462             :  * @param mmapMetaData to verify
     463             :  *
     464             :  * @retval 0 if magic MmapMetaData is consistent
     465             :  * @retval -1 if magic MmapMetaData is inconsistent
     466             :  */
     467             : static int verifyMagicMmapMetaData (MmapMetaData * mmapMetaData)
     468             : {
     469         223 :         if (!mmapMetaData) return -1;
     470         223 :         return memcmp (mmapMetaData, &magicMmapMetaData, SIZEOF_MMAPMETADATA);
     471             : }
     472             : 
     473             : /**
     474             :  * @brief Verify magic data in the mapped region.
     475             :  *
     476             :  * @param mappedRegion pointer to mapped region
     477             :  *
     478             :  * @retval 0 if magic data is consistent
     479             :  * @retval -1 if magic data is inconsistent
     480             :  */
     481         225 : static int verifyMagicData (char * mappedRegion)
     482             : {
     483         225 :         KeySet * destKeySet = (KeySet *) (mappedRegion + OFFSET_MAGIC_KEYSET);
     484         225 :         Key * keyPtr = (Key *) (mappedRegion + OFFSET_MAGIC_KEY);
     485         225 :         MmapMetaData * mmapMetaData = (MmapMetaData *) (mappedRegion + OFFSET_MAGIC_MMAPMETADATA);
     486             : 
     487         673 :         if ((verifyMagicKey (keyPtr) != 0) || (verifyMagicKeySet (destKeySet) != 0) || (verifyMagicMmapMetaData (mmapMetaData) != 0))
     488             :         {
     489             :                 return -1;
     490             :         }
     491             : 
     492             :         return 0;
     493             : }
     494             : 
     495             : #ifdef ELEKTRA_MMAP_CHECKSUM
     496             : /**
     497             :  * @brief Verify checksum of the critical mmap data.
     498             :  *
     499             :  * Verifies the CRC32 checksum of all KeySet and Key structs (including pointers/pointer arrays)
     500             :  * as well as the MmapMetaData. Does not check Key name and value.
     501             :  *
     502             :  * @param mappedRegion pointer to mapped region
     503             :  * @param mmapHeader containing the stored checksum and size of the checksum region
     504             :  *
     505             :  * @retval 0 if checksum was correct
     506             :  * @retval -1 if there was a checksum mismatch
     507             :  */
     508          76 : static int verifyChecksum (char * mappedRegion, MmapHeader * mmapHeader, PluginMode mode)
     509             : {
     510             :         // if file was written without checksum, we skip the check
     511          76 :         if (!test_bit (mmapHeader->formatFlags, MMAP_FLAG_CHECKSUM)) return 0;
     512             : 
     513          74 :         uint32_t checksum = crc32 (0L, Z_NULL, 0);
     514          74 :         checksum = crc32 (checksum, (const unsigned char *) (mappedRegion + SIZEOF_MMAPHEADER), mmapHeader->cksumSize);
     515             : 
     516          74 :         if (checksum != mmapHeader->checksum)
     517             :         {
     518             :                 ELEKTRA_MMAP_LOG_WARNING ("old checksum: %ul", mmapHeader->checksum);
     519             :                 ELEKTRA_MMAP_LOG_WARNING ("new checksum: %ul", checksum);
     520             :                 return -1;
     521             :         }
     522             :         return 0;
     523             : }
     524             : #endif
     525             : 
     526             : /**
     527             :  * @brief Calculates the size, in bytes, needed to store the KeySet in a mmap region.
     528             :  *
     529             :  * Iterates over the KeySet and calculates the complete size in bytes, needed to store the KeySet
     530             :  * within a mapped region. The size includes all mmap meta-information, magic KeySet and Key for
     531             :  * consistency checks, KeySets, meta-KeySets, Keys, meta-Keys, Key names and values.
     532             :  * Copied meta-Keys are counted once for deduplication. If needed, padding is added to align the
     533             :  * MmapFooter properly at the end of the mapping. When the plugin is in a global position,
     534             :  * acting as a cache, the calculated size includes the global KeySet.
     535             :  *
     536             :  * The complete size and some other meta-information are stored in the MmapHeader and MmapMetaData.
     537             :  * The DynArray stores the unique meta-Key pointers needed for deduplication.
     538             :  *
     539             :  * @param mmapHeader to store the allocation size
     540             :  * @param mmapMetaData to store the number of KeySets and Keys
     541             :  * @param returned the KeySet that should be stored
     542             :  * @param global the global KeySet
     543             :  * @param dynArray to store meta-key pointers for deduplication
     544             :  */
     545         344 : static void calculateMmapDataSize (MmapHeader * mmapHeader, MmapMetaData * mmapMetaData, KeySet * returned, KeySet * global,
     546             :                                    DynArray * dynArray)
     547             : {
     548             :         Key * cur;
     549         344 :         ksRewind (returned);
     550         344 :         size_t dataBlocksSize = 0; // sum of keyName and keyValue sizes
     551         344 :         mmapMetaData->numKeys = 0;
     552         344 :         mmapMetaData->numKeySets = 3;                 // include the magic, global and main keyset
     553         344 :         mmapMetaData->ksAlloc = returned->alloc; // sum of allocation sizes for all meta-keysets
     554        1623 :         while ((cur = ksNext (returned)) != 0)
     555             :         {
     556        1279 :                 dataBlocksSize += (cur->keySize + cur->keyUSize + cur->dataSize);
     557             : 
     558        1279 :                 if (cur->meta)
     559             :                 {
     560         295 :                         ++mmapMetaData->numKeySets;
     561             : 
     562             :                         Key * curMeta;
     563         295 :                         ksRewind (cur->meta);
     564         590 :                         while ((curMeta = ksNext (cur->meta)) != 0)
     565             :                         {
     566         295 :                                 if (ELEKTRA_PLUGIN_FUNCTION (dynArrayFindOrInsert) (curMeta, dynArray) == 0)
     567             :                                 {
     568             :                                         // key was just inserted
     569         291 :                                         dataBlocksSize += (curMeta->keySize + curMeta->keyUSize + curMeta->dataSize);
     570             :                                 }
     571             :                         }
     572         295 :                         mmapMetaData->ksAlloc += (cur->meta->alloc);
     573             :                 }
     574             :         }
     575             : 
     576         344 :         if (global) // TODO: remove this code duplication
     577             :         {
     578             :                 ELEKTRA_LOG_DEBUG ("calculate global keyset into size");
     579          18 :                 mmapMetaData->ksAlloc += global->alloc;
     580          18 :                 mmapMetaData->numKeys += global->size;
     581             : 
     582             :                 Key * globalKey;
     583          18 :                 ksRewind (global);
     584          34 :                 while ((globalKey = ksNext (global)) != 0)
     585             :                 {
     586          16 :                         dataBlocksSize += (globalKey->keySize + globalKey->keyUSize + globalKey->dataSize);
     587             : 
     588          16 :                         if (globalKey->meta)
     589             :                         {
     590          10 :                                 ++mmapMetaData->numKeySets;
     591             : 
     592             :                                 Key * curMeta;
     593          10 :                                 ksRewind (globalKey->meta);
     594          20 :                                 while ((curMeta = ksNext (globalKey->meta)) != 0)
     595             :                                 {
     596          10 :                                         if (ELEKTRA_PLUGIN_FUNCTION (dynArrayFindOrInsert) (curMeta, dynArray) == 0)
     597             :                                         {
     598             :                                                 // key was just inserted
     599          10 :                                                 dataBlocksSize += (curMeta->keySize + curMeta->keyUSize + curMeta->dataSize);
     600             :                                         }
     601             :                                 }
     602          10 :                                 mmapMetaData->ksAlloc += (globalKey->meta->alloc);
     603             :                         }
     604             :                 }
     605             :         }
     606         344 :         mmapMetaData->numKeys += returned->size + dynArray->size + 1; // +1 for magic Key
     607             : 
     608         344 :         size_t keyArraySize = mmapMetaData->numKeys * SIZEOF_KEY;
     609             :         mmapHeader->allocSize =
     610         344 :                 (SIZEOF_KEYSET * mmapMetaData->numKeySets) + keyArraySize + dataBlocksSize + (mmapMetaData->ksAlloc * SIZEOF_KEY_PTR);
     611         344 :         mmapHeader->cksumSize = mmapHeader->allocSize + (SIZEOF_MMAPMETADATA * 2); // cksumSize now contains size of all critical data
     612             : 
     613         344 :         size_t padding = sizeof (uint64_t) - (mmapHeader->allocSize % sizeof (uint64_t)); // alignment for MMAP Footer at end of mapping
     614         344 :         mmapHeader->allocSize += SIZEOF_MMAPHEADER + (SIZEOF_MMAPMETADATA * 2) + SIZEOF_MMAPFOOTER + padding;
     615             : 
     616         344 :         mmapMetaData->numKeys--;    // don't include magic Key
     617         344 :         mmapMetaData->numKeySets--; // don't include magic KeySet
     618         344 : }
     619             : 
     620             : /**
     621             :  * @brief Write meta-keys to the mapped region.
     622             :  *
     623             :  * This function only writes all meta-keys to the mapped region.
     624             :  * The meta-keys are later referenced in the meta-keysets.
     625             :  *
     626             :  * @param mmapAddr struct containing pointers to the mapped region
     627             :  * @param dynArray containing deduplicated meta-keys
     628             :  */
     629         344 : static void writeMetaKeys (MmapAddr * mmapAddr, DynArray * dynArray)
     630             : {
     631         344 :         if (dynArray->size == 0)
     632             :         {
     633             :                 return;
     634             :         }
     635             : 
     636             :         // allocate space in DynArray to remember the addresses of mapped meta-keys
     637          97 :         dynArray->mappedKeyArray = elektraCalloc (dynArray->size * sizeof (Key *));
     638             : 
     639             :         // write the meta keys into place
     640         398 :         for (size_t i = 0; i < dynArray->size; ++i)
     641             :         {
     642         301 :                 Key * curMeta = dynArray->keyArray[i];       // old key location
     643         301 :                 Key * mmapMetaKey = (Key *) mmapAddr->keyPtr; // new key location
     644         301 :                 *mmapMetaKey = *curMeta;
     645         301 :                 mmapAddr->keyPtr += SIZEOF_KEY;
     646             : 
     647             :                 // move Key name
     648         301 :                 if (curMeta->key)
     649             :                 {
     650         301 :                         size_t keyNameSize = curMeta->keySize + curMeta->keyUSize;
     651         301 :                         memcpy (mmapAddr->dataPtr, curMeta->key, keyNameSize);
     652         301 :                         mmapMetaKey->key = mmapAddr->dataPtr - mmapAddr->mmapAddrInt;
     653         301 :                         mmapAddr->dataPtr += keyNameSize;
     654         301 :                         mmapMetaKey->flags |= KEY_FLAG_MMAP_KEY;
     655             :                 }
     656             : 
     657             :                 // move Key value
     658         301 :                 if (curMeta->data.v)
     659             :                 {
     660         301 :                         memcpy (mmapAddr->dataPtr, curMeta->data.v, curMeta->dataSize);
     661         301 :                         mmapMetaKey->data.v = mmapAddr->dataPtr - mmapAddr->mmapAddrInt;
     662         301 :                         mmapAddr->dataPtr += curMeta->dataSize;
     663         301 :                         mmapMetaKey->flags |= KEY_FLAG_MMAP_DATA;
     664             :                 }
     665             : 
     666             :                 // move Key itself
     667         301 :                 mmapMetaKey->flags |= KEY_FLAG_MMAP_STRUCT;
     668         301 :                 mmapMetaKey->meta = 0;
     669         301 :                 mmapMetaKey->ksReference = 0;
     670             : 
     671         301 :                 dynArray->mappedKeyArray[i] = mmapMetaKey;
     672             :         }
     673             : }
     674             : 
     675             : /**
     676             :  * @brief Writes a meta keyset of a key to the mapped region.
     677             :  *
     678             :  * @param key holding the meta-keyset
     679             :  * @param mmapAddr structure holding pointers to the mapped region
     680             :  * @param dynArray holding deduplicated references to meta-keys
     681             :  *
     682             :  * @return pointer to the new meta keyset
     683             :  */
     684        1295 : static KeySet * writeMetaKeySet (Key * key, MmapAddr * mmapAddr, DynArray * dynArray)
     685             : {
     686             :         // write the meta KeySet
     687        1295 :         if (!key->meta) return 0;
     688             : 
     689         305 :         KeySet * newMeta = (KeySet *) mmapAddr->metaKsPtr;
     690         305 :         mmapAddr->metaKsPtr += SIZEOF_KEYSET;
     691             : 
     692         305 :         newMeta->flags = key->meta->flags | KS_FLAG_MMAP_STRUCT | KS_FLAG_MMAP_ARRAY;
     693         305 :         newMeta->array = (Key **) mmapAddr->metaKsArrayPtr;
     694         305 :         mmapAddr->metaKsArrayPtr += SIZEOF_KEY_PTR * key->meta->alloc;
     695             : 
     696         305 :         keyRewindMeta (key);
     697         305 :         size_t metaKeyIndex = 0;
     698         305 :         Key * mappedMetaKey = 0;
     699             :         const Key * metaKey;
     700         915 :         while ((metaKey = keyNextMeta (key)) != 0)
     701             :         {
     702             :                 // get address of mapped key and store it in the new array
     703         305 :                 mappedMetaKey = dynArray->mappedKeyArray[ELEKTRA_PLUGIN_FUNCTION (dynArrayFind) ((Key *) metaKey, dynArray)];
     704         305 :                 newMeta->array[metaKeyIndex] = (Key *) ((char *) mappedMetaKey - mmapAddr->mmapAddrInt);
     705         305 :                 if (mappedMetaKey->ksReference < SSIZE_MAX)
     706             :                 {
     707         305 :                         ++(mappedMetaKey->ksReference);
     708             :                 }
     709         305 :                 ++metaKeyIndex;
     710             :         }
     711         305 :         newMeta->array[key->meta->size] = 0;
     712         305 :         newMeta->array = (Key **) ((char *) newMeta->array - mmapAddr->mmapAddrInt);
     713         305 :         newMeta->alloc = key->meta->alloc;
     714         305 :         newMeta->size = key->meta->size;
     715         305 :         newMeta = (KeySet *) ((char *) newMeta - mmapAddr->mmapAddrInt);
     716         305 :         return newMeta;
     717             : }
     718             : 
     719             : /**
     720             :  * @brief Writes the keys of the keyset to the mapped region.
     721             :  *
     722             :  * For each key, the corresponding meta keyset is also written
     723             :  * to the mapped region.
     724             :  *
     725             :  * @param keySet holding the keys to be written to the mapped region
     726             :  * @param mmapAddr structure holding pointers to the mapped region
     727             :  * @param dynArray holding deduplicated meta-key pointers
     728             :  */
     729         202 : static void writeKeys (KeySet * keySet, MmapAddr * mmapAddr, DynArray * dynArray, PluginMode mode)
     730             : {
     731             :         Key * cur;
     732         202 :         size_t keyIndex = 0;
     733             : 
     734         202 :         Key ** ksArray = 0;
     735         202 :         if (test_bit (mode, MODE_STORAGE))
     736             :         {
     737         198 :                 ksArray = (Key **) mmapAddr->ksArrayPtr;
     738             :         }
     739             :         else
     740             :         {
     741           4 :                 ksArray = (Key **) mmapAddr->globalKsArrayPtr;
     742             :         }
     743             : 
     744         202 :         ksRewind (keySet);
     745        1699 :         while ((cur = ksNext (keySet)) != 0)
     746             :         {
     747        1295 :                 Key * mmapKey = (Key *) mmapAddr->keyPtr; // new key location
     748        1295 :                 mmapAddr->keyPtr += SIZEOF_KEY;
     749        1295 :                 *mmapKey = *cur;
     750             : 
     751             :                 // move Key name
     752        1295 :                 if (cur->key)
     753             :                 {
     754        1295 :                         size_t keyNameSize = cur->keySize + cur->keyUSize;
     755        1295 :                         memcpy (mmapAddr->dataPtr, cur->key, keyNameSize);
     756        1295 :                         mmapKey->key = mmapAddr->dataPtr - mmapAddr->mmapAddrInt;
     757        1295 :                         mmapAddr->dataPtr += keyNameSize;
     758        1295 :                         mmapKey->flags |= KEY_FLAG_MMAP_KEY;
     759             :                 }
     760             : 
     761             :                 // move Key value
     762        1295 :                 if (cur->data.v)
     763             :                 {
     764        1291 :                         memcpy (mmapAddr->dataPtr, cur->data.v, cur->dataSize);
     765        1291 :                         mmapKey->data.v = mmapAddr->dataPtr - mmapAddr->mmapAddrInt;
     766        1291 :                         mmapAddr->dataPtr += cur->dataSize;
     767        1291 :                         mmapKey->flags |= KEY_FLAG_MMAP_DATA;
     768             :                 }
     769             :                 else
     770             :                 {
     771           4 :                         mmapKey->data.v = 0;
     772           4 :                         mmapKey->dataSize = 0;
     773             :                 }
     774             : 
     775             :                 // write the meta KeySet
     776        1295 :                 mmapKey->meta = writeMetaKeySet (cur, mmapAddr, dynArray);
     777             : 
     778             :                 // move Key itself
     779        1295 :                 mmapKey->flags |= KEY_FLAG_MMAP_STRUCT;
     780        1295 :                 mmapKey->ksReference = 1;
     781             : 
     782             :                 // write the relative Key pointer into the KeySet array
     783        1295 :                 ksArray[keyIndex] = (Key *) ((char *) mmapKey - mmapAddr->mmapAddrInt);
     784             : 
     785        1295 :                 ++keyIndex;
     786             :         }
     787         202 : }
     788             : 
     789             : static void printMmapAddr (MmapAddr * mmapAddr ELEKTRA_UNUSED)
     790             : {
     791             :         ELEKTRA_LOG_DEBUG ("globalKsPtr: \t\t %p", (void *) mmapAddr->globalKsPtr);
     792             :         ELEKTRA_LOG_DEBUG ("ksPtr: \t\t %p", (void *) mmapAddr->ksPtr);
     793             :         ELEKTRA_LOG_DEBUG ("metaKsPtr: \t\t %p", (void *) mmapAddr->metaKsPtr);
     794             :         ELEKTRA_LOG_DEBUG ("globalKsArrayPtr: \t %p", (void *) mmapAddr->globalKsArrayPtr);
     795             :         ELEKTRA_LOG_DEBUG ("ksArrayPtr: \t\t %p", (void *) mmapAddr->ksArrayPtr);
     796             :         ELEKTRA_LOG_DEBUG ("metaKsArrayPtr: \t %p", (void *) mmapAddr->metaKsArrayPtr);
     797             :         ELEKTRA_LOG_DEBUG ("keyPtr: \t\t %p", (void *) mmapAddr->keyPtr);
     798             :         ELEKTRA_LOG_DEBUG ("dataPtr: \t\t %p", (void *) mmapAddr->dataPtr);
     799             :         ELEKTRA_LOG_DEBUG ("mmapAddrInt: \t\t %lu", mmapAddr->mmapAddrInt);
     800             : }
     801             : 
     802             : static void printMmapMetaData (MmapMetaData * mmapMetaData ELEKTRA_UNUSED)
     803             : {
     804             :         ELEKTRA_LOG_DEBUG ("numKeySets: \t %lu", mmapMetaData->numKeySets);
     805             :         ELEKTRA_LOG_DEBUG ("ksAlloc: \t %lu", mmapMetaData->ksAlloc);
     806             :         ELEKTRA_LOG_DEBUG ("numKeys: \t %lu", mmapMetaData->numKeys);
     807             : }
     808             : 
     809             : 
     810             : /**
     811             :  * @brief Copies a keyset to a mapped region.
     812             :  *
     813             :  * Writes a keyset with all needed information to the mapped region and calculates
     814             :  * the checksum. Everything is written to the mapped region, including keyset, keys,
     815             :  * meta-information and data for consistency checks.
     816             :  *
     817             :  * @param dest address of the mapped region
     818             :  * @param keySet to copy to the mapped region
     819             :  * @param global global keyset
     820             :  * @param mmapHeader containing the size and checksum of the mapped region
     821             :  * @param mmapMetaData containing meta-information of the mapped region
     822             :  * @param mmapFooter containing a magic number for consistency checks
     823             :  * @param dynArray containing deduplicated pointers to meta-keys
     824             :  */
     825         344 : static void copyKeySetToMmap (char * const dest, KeySet * keySet, KeySet * global, MmapHeader * mmapHeader, MmapMetaData * mmapMetaData,
     826             :                               MmapFooter * mmapFooter, DynArray * dynArray)
     827             : {
     828         344 :         writeMagicData (dest);
     829             : 
     830         344 :         size_t globalAlloc = 0;
     831         344 :         if (global) globalAlloc = global->alloc;
     832             : 
     833        3096 :         MmapAddr mmapAddr = { .globalKsPtr = (KeySet *) (dest + OFFSET_GLOBAL_KEYSET),
     834         344 :                               .ksPtr = (KeySet *) (dest + OFFSET_KEYSET),
     835         344 :                               .metaKsPtr = (char *) mmapAddr.ksPtr + SIZEOF_KEYSET,
     836         344 :                               .globalKsArrayPtr = (dest + OFFSET_GLOBAL_KEYSET) + (SIZEOF_KEYSET * mmapMetaData->numKeySets),
     837         344 :                               .ksArrayPtr = mmapAddr.globalKsArrayPtr + (SIZEOF_KEY_PTR * globalAlloc),
     838         344 :                               .metaKsArrayPtr = mmapAddr.ksArrayPtr + (SIZEOF_KEY_PTR * keySet->alloc),
     839         344 :                               .keyPtr = mmapAddr.globalKsArrayPtr + (SIZEOF_KEY_PTR * mmapMetaData->ksAlloc),
     840         344 :                               .dataPtr = mmapAddr.keyPtr + (SIZEOF_KEY * mmapMetaData->numKeys),
     841         344 :                               .mmapAddrInt = (uintptr_t) dest };
     842             : 
     843         344 :         printMmapAddr (&mmapAddr);
     844         344 :         printMmapMetaData (mmapMetaData);
     845             : 
     846             :         // first write the meta keys into place
     847         344 :         writeMetaKeys (&mmapAddr, dynArray);
     848             : 
     849         344 :         if (global)
     850             :         {
     851             :                 ELEKTRA_LOG_DEBUG ("writing GLOBAL KEYSET");
     852          18 :                 if (global->size != 0) writeKeys (global, &mmapAddr, dynArray, MODE_GLOBALCACHE);
     853             : 
     854          18 :                 set_bit (mmapHeader->formatFlags, MMAP_FLAG_TIMESTAMPS);
     855          18 :                 mmapAddr.globalKsPtr->flags = global->flags | KS_FLAG_MMAP_STRUCT | KS_FLAG_MMAP_ARRAY;
     856          18 :                 mmapAddr.globalKsPtr->array = (Key **) mmapAddr.globalKsArrayPtr;
     857          18 :                 mmapAddr.globalKsPtr->array[global->size] = 0;
     858          18 :                 mmapAddr.globalKsPtr->array = (Key **) (mmapAddr.globalKsArrayPtr - mmapAddr.mmapAddrInt);
     859          18 :                 mmapAddr.globalKsPtr->alloc = global->alloc;
     860          18 :                 mmapAddr.globalKsPtr->size = global->size;
     861             :         }
     862             : 
     863         344 :         if (keySet->size != 0)
     864             :         {
     865             :                 // now write Keys including meta KeySets
     866         198 :                 writeKeys (keySet, &mmapAddr, dynArray, MODE_STORAGE);
     867             :         }
     868             : 
     869         344 :         mmapAddr.ksPtr->flags = keySet->flags | KS_FLAG_MMAP_STRUCT | KS_FLAG_MMAP_ARRAY;
     870         344 :         mmapAddr.ksPtr->array = (Key **) mmapAddr.ksArrayPtr;
     871         344 :         mmapAddr.ksPtr->array[keySet->size] = 0;
     872         344 :         mmapAddr.ksPtr->array = (Key **) (mmapAddr.ksArrayPtr - mmapAddr.mmapAddrInt);
     873         344 :         mmapAddr.ksPtr->alloc = keySet->alloc;
     874         344 :         mmapAddr.ksPtr->size = keySet->size;
     875             : 
     876         344 :         memcpy ((dest + OFFSET_MMAPMETADATA), mmapMetaData, SIZEOF_MMAPMETADATA);
     877             : #ifdef ELEKTRA_MMAP_CHECKSUM
     878         132 :         uint32_t checksum = crc32 (0L, Z_NULL, 0);
     879         132 :         checksum = crc32 (checksum, (const unsigned char *) (dest + SIZEOF_MMAPHEADER), mmapHeader->cksumSize);
     880         132 :         mmapHeader->checksum = checksum;
     881             : #endif
     882         344 :         memcpy (dest, mmapHeader, SIZEOF_MMAPHEADER);
     883         344 :         memcpy ((dest + mmapHeader->allocSize - SIZEOF_MMAPFOOTER), mmapFooter, SIZEOF_MMAPFOOTER);
     884         344 : }
     885             : 
     886             : /**
     887             :  * @brief Replaces contents of a keyset with the keyset from the mapped region.
     888             :  *
     889             :  * The keyset members are replaced with data from a mapped keyset. The keyset
     890             :  * array then points into the mapped region.
     891             :  *
     892             :  * @param mappedRegion pointer to mapped region, holding an already written keyset
     893             :  * @param returned keyset to be replaced by the mapped keyset
     894             :  */
     895         223 : static void mmapToKeySet (Plugin * handle, char * mappedRegion, KeySet * returned, PluginMode mode)
     896             : {
     897         223 :         KeySet * keySet = (KeySet *) (mappedRegion + OFFSET_KEYSET);
     898         223 :         ksClose (returned);
     899         223 :         returned->array = keySet->array;
     900         223 :         returned->size = keySet->size;
     901         223 :         returned->alloc = keySet->alloc;
     902             :         // to be able to free() the returned KeySet, just set the array flag here
     903         223 :         returned->flags = KS_FLAG_MMAP_ARRAY;
     904             :         // we intentionally do not change the KeySet->opmphm here!
     905             : 
     906         223 :         if (test_bit (mode, MODE_GLOBALCACHE)) // TODO: remove code duplication here
     907             :         {
     908          41 :                 KeySet * global = elektraPluginGetGlobalKeySet (handle);
     909          41 :                 ksClose (global); // TODO: if we have a global keyset, maybe use ksAppend?
     910          41 :                 KeySet * mmapTimeStamps = (KeySet *) (mappedRegion + OFFSET_GLOBAL_KEYSET);
     911          41 :                 global->array = mmapTimeStamps->array;
     912          41 :                 global->size = mmapTimeStamps->size;
     913          41 :                 global->alloc = mmapTimeStamps->alloc;
     914             :                 // to be able to free() the timeStamps KeySet, just set the array flag here
     915          41 :                 global->flags = KS_FLAG_MMAP_ARRAY;
     916             :                 // we intentionally do not change the KeySet->opmphm here!
     917             :         }
     918         223 : }
     919             : 
     920             : /**
     921             :  * @brief Updates pointers of a mapped keyset to a new location in memory.
     922             :  *
     923             :  * After mapping a file to a new location, all pointers have to be updated
     924             :  * in order to be consistent. When the mapped keyset is written, we subtract
     925             :  * the base address. Therefore, after mapping the keyset to a new memory location,
     926             :  * we only have to add the new base address to all pointers.
     927             :  *
     928             :  * @param mmapMetaData meta-data of the old mapped region
     929             :  * @param dest new mapped memory region
     930             :  */
     931         223 : static void updatePointers (MmapMetaData * mmapMetaData, char * dest)
     932             : {
     933         223 :         uintptr_t destInt = (uintptr_t) dest;
     934             : 
     935         223 :         char * ksPtr = (dest + OFFSET_GLOBAL_KEYSET);
     936         223 :         char * ksArrayPtr = ksPtr + SIZEOF_KEYSET * mmapMetaData->numKeySets;
     937         223 :         char * keyPtr = ksArrayPtr + SIZEOF_KEY_PTR * mmapMetaData->ksAlloc;
     938             : 
     939         997 :         for (size_t i = 0; i < mmapMetaData->numKeySets; ++i)
     940             :         {
     941         774 :                 KeySet * ks = (KeySet *) ksPtr;
     942         774 :                 ksPtr += SIZEOF_KEYSET;
     943         774 :                 if (ks->array)
     944             :                 {
     945         592 :                         ks->array = (Key **) ((char *) ks->array + destInt);
     946        2034 :                         for (size_t j = 0; j < ks->size; ++j)
     947             :                         {
     948        1442 :                                 ks->array[j] = (Key *) ((char *) (ks->array[j]) + destInt);
     949             :                         }
     950             :                 }
     951             :         }
     952             : 
     953        1438 :         for (size_t i = 0; i < mmapMetaData->numKeys; ++i)
     954             :         {
     955        1438 :                 Key * key = (Key *) keyPtr;
     956        1438 :                 keyPtr += SIZEOF_KEY;
     957        1438 :                 if (key->data.c)
     958             :                 {
     959        1434 :                         key->data.c = key->data.c + destInt;
     960             :                 }
     961        1438 :                 if (key->key)
     962             :                 {
     963        1438 :                         key->key = key->key + destInt;
     964             :                 }
     965        1438 :                 if (key->meta)
     966             :                 {
     967         328 :                         key->meta = (KeySet *) ((char *) (key->meta) + destInt);
     968             :                 }
     969             :         }
     970         223 : }
     971             : 
     972             : /* -- Exported Elektra Plugin Functions ------------------------------------------------------------------------------------------------- */
     973             : 
     974             : /**
     975             :  * @brief Initializes the plugin data and magic keyset and key.
     976             :  *
     977             :  * @param handle The plugin handle.
     978             :  * @param errorKey Unused.
     979             :  *
     980             :  * @retval ELEKTRA_PLUGIN_STATUS_ERROR on memory error (plugin data (keyset) could not be allocated).
     981             :  * @retval ELEKTRA_PLUGIN_STATUS_SUCCESS if initialization was successful.
     982             :  */
     983         608 : int ELEKTRA_PLUGIN_FUNCTION (open) (Plugin * handle ELEKTRA_UNUSED, Key * errorKey ELEKTRA_UNUSED)
     984             : {
     985             :         // plugin initialization logic
     986             : 
     987         608 :         const uintptr_t magicNumber = generateMagicNumber ();
     988         608 :         if (magicKeySet.array == 0) initMagicKeySet (magicNumber);
     989         608 :         if (magicKey.data.v == 0) initMagicKey (magicNumber);
     990         608 :         if (magicMmapMetaData.numKeys == 0) initMagicMmapMetaData ();
     991             : 
     992         608 :         return ELEKTRA_PLUGIN_STATUS_SUCCESS;
     993             : }
     994             : 
     995             : /**
     996             :  * @brief Cleans up the plugin data and unlinks all mapped files.
     997             :  *
     998             :  * @param handle The plugin handle.
     999             :  * @param errorKey Unused.
    1000             :  *
    1001             :  * @retval ELEKTRA_PLUGIN_STATUS_SUCCESS always.
    1002             :  */
    1003         608 : int ELEKTRA_PLUGIN_FUNCTION (close) (Plugin * handle ELEKTRA_UNUSED, Key * errorKey ELEKTRA_UNUSED)
    1004             : {
    1005             :         // free all plugin resources and shut it down
    1006             : 
    1007         608 :         return ELEKTRA_PLUGIN_STATUS_SUCCESS;
    1008             : }
    1009             : 
    1010             : /**
    1011             :  * @brief The mmapstorage get function loads a keyset from a file and returns it.
    1012             :  *
    1013             :  * On successful mapping the returned keyset is replaced by the mapped keyset.
    1014             :  * The returned keyset array then points to a mapped region.
    1015             :  *
    1016             :  * @param handle The plugin handle.
    1017             :  * @param ks The keyset which is replaced by the mapped keyset.
    1018             :  * @param parentKey Holding the filename or error message.
    1019             :  *
    1020             :  * @retval ELEKTRA_PLUGIN_STATUS_SUCCESS if the file was mapped successfully.
    1021             :  * @retval ELEKTRA_PLUGIN_STATUS_ERROR if the file could not be mapped successfully.
    1022             :  */
    1023         325 : int ELEKTRA_PLUGIN_FUNCTION (get) (Plugin * handle ELEKTRA_UNUSED, KeySet * ks, Key * parentKey)
    1024             : {
    1025             :         // get all keys
    1026         325 :         int errnosave = errno;
    1027         325 :         PluginMode mode = MODE_STORAGE;
    1028             : 
    1029         325 :         if (elektraPluginGetGlobalKeySet (handle) != 0)
    1030             :         {
    1031             :                 ELEKTRA_LOG_DEBUG ("mmapstorage global position called");
    1032          41 :                 mode = MODE_GLOBALCACHE;
    1033             :         }
    1034             : 
    1035         325 :         Key * root = keyNew ("system/elektra/modules/" ELEKTRA_PLUGIN_NAME, KEY_END);
    1036         325 :         if (keyRel (root, parentKey) >= 0)
    1037             :         {
    1038          88 :                 keyDel (root);
    1039          88 :                 KeySet * contract =
    1040             : #include "contract.h"
    1041          88 :                         ksAppend (ks, contract);
    1042          88 :                 ksDel (contract);
    1043          88 :                 return ELEKTRA_PLUGIN_STATUS_SUCCESS;
    1044             :         }
    1045         237 :         keyDel (root);
    1046             : 
    1047         237 :         int fd = -1;
    1048         237 :         char * mappedRegion = MAP_FAILED;
    1049         237 :         Key * initialParent = keyDup (parentKey);
    1050             : 
    1051         237 :         if (elektraStrCmp (keyString (parentKey), STDIN_FILENAME) == 0)
    1052             :         {
    1053           2 :                 fd = fileno (stdin);
    1054             :                 ELEKTRA_LOG_DEBUG ("MODE_NONREGULAR_FILE");
    1055           2 :                 set_bit (mode, MODE_NONREGULAR_FILE);
    1056             :         }
    1057         470 :         else if ((fd = openFile (parentKey, O_RDONLY, 0, mode)) == -1)
    1058             :         {
    1059             :                 goto error;
    1060             :         }
    1061             : 
    1062             :         struct stat sbuf;
    1063         235 :         if (fstatFile (fd, &sbuf, parentKey, mode) != 1)
    1064             :         {
    1065             :                 goto error;
    1066             :         }
    1067             : 
    1068         235 :         if (test_bit (mode, MODE_NONREGULAR_FILE))
    1069             :         {
    1070             :                 // non regular file not mmap compatible, copy to temp file
    1071             :                 ELEKTRA_LOG_DEBUG ("copy nonregular file");
    1072           2 :                 if ((fd = copyToAnonymousTempfile (fd, &sbuf, parentKey, mode)) < 0)
    1073             :                 {
    1074             :                         goto error;
    1075             :                 }
    1076           2 :                 clear_bit (mode, MODE_NONREGULAR_FILE);
    1077             :         }
    1078             : 
    1079         235 :         if (sbuf.st_size == 0)
    1080             :         {
    1081             :                 // empty mmap file
    1082           2 :                 if (close (fd) != 0)
    1083             :                 {
    1084             :                         ELEKTRA_MMAP_LOG_WARNING ("could not close");
    1085             :                         goto error;
    1086             :                 }
    1087           2 :                 keySetString (parentKey, keyString (initialParent));
    1088           2 :                 if (initialParent) keyDel (initialParent);
    1089             :                 return ELEKTRA_PLUGIN_STATUS_SUCCESS;
    1090             :         }
    1091             : 
    1092         233 :         if (sbuf.st_size < 0 || (size_t) sbuf.st_size < ELEKTRA_MMAP_MINSIZE)
    1093             :         {
    1094             :                 // file is smaller than minimum size
    1095             :                 goto error;
    1096             :         }
    1097             : 
    1098         233 :         mappedRegion = mmapFile ((void *) 0, fd, sbuf.st_size, MAP_PRIVATE, parentKey, mode);
    1099         233 :         if (mappedRegion == MAP_FAILED)
    1100             :         {
    1101             :                 ELEKTRA_MMAP_LOG_WARNING ("mappedRegion == MAP_FAILED");
    1102             :                 goto error;
    1103             :         }
    1104             : 
    1105             :         MmapHeader * mmapHeader;
    1106             :         MmapMetaData * mmapMetaData;
    1107         233 :         if (readHeader (mappedRegion, &mmapHeader, &mmapMetaData) == -1)
    1108             :         {
    1109             :                 // config file was corrupt
    1110             :                 ELEKTRA_MMAP_LOG_WARNING ("could not read mmap information header");
    1111             :                 goto error;
    1112             :         }
    1113             : 
    1114             :         if (!test_bit (mmapHeader->formatFlags, MMAP_FLAG_TIMESTAMPS) && mode == MODE_GLOBALCACHE)
    1115             :         {
    1116             :                 ELEKTRA_MMAP_LOG_WARNING ("plugin in global cache mode, but file does not contain timestamps");
    1117             :         }
    1118             : 
    1119         229 :         if (sbuf.st_size < 0 || (size_t) sbuf.st_size != mmapHeader->allocSize)
    1120             :         {
    1121             :                 // config file size mismatch
    1122             :                 ELEKTRA_MMAP_LOG_WARNING ("mmap file size differs from metadata, file was altered");
    1123             :                 goto error;
    1124             :         }
    1125             : 
    1126         454 :         if (readFooter (mappedRegion, mmapHeader) == -1)
    1127             :         {
    1128             :                 // config file was corrupt/truncated
    1129             :                 ELEKTRA_MMAP_LOG_WARNING ("could not read mmap information footer: file was altered");
    1130             :                 goto error;
    1131             :         }
    1132             : 
    1133             : #ifdef ELEKTRA_MMAP_CHECKSUM
    1134          76 :         if (verifyChecksum (mappedRegion, mmapHeader, mode) != 0)
    1135             :         {
    1136             :                 ELEKTRA_MMAP_LOG_WARNING ("checksum failed");
    1137             :                 goto error;
    1138             :         }
    1139             : #endif
    1140             : 
    1141         225 :         if (verifyMagicData (mappedRegion) != 0)
    1142             :         {
    1143             :                 // magic data could not be read properly, indicating unreadable format or different architecture
    1144             :                 ELEKTRA_MMAP_LOG_WARNING ("mmap magic data could not be read properly");
    1145             :                 goto error;
    1146             :         }
    1147             : 
    1148         223 :         updatePointers (mmapMetaData, mappedRegion);
    1149         223 :         mmapToKeySet (handle, mappedRegion, ks, mode);
    1150             : 
    1151         223 :         if (close (fd) != 0)
    1152             :         {
    1153             :                 ELEKTRA_MMAP_LOG_WARNING ("could not close");
    1154             :                 goto error;
    1155             :         }
    1156             : 
    1157         223 :         keySetString (parentKey, keyString (initialParent));
    1158         223 :         if (initialParent) keyDel (initialParent);
    1159             :         return ELEKTRA_PLUGIN_STATUS_SUCCESS;
    1160             : 
    1161             : error:
    1162          12 :         if (errno != 0)
    1163             :         {
    1164             :                 ELEKTRA_MMAP_LOG_WARNING ("strerror: %s", strerror (errno));
    1165           2 :                 ELEKTRA_SET_ERROR_GET (parentKey);
    1166             :         }
    1167             : 
    1168          12 :         if ((mappedRegion != MAP_FAILED) && (munmap (mappedRegion, sbuf.st_size) != 0))
    1169             :         {
    1170             :                 ELEKTRA_MMAP_LOG_WARNING ("could not munmap");
    1171             :         }
    1172          12 :         if ((fd != -1) && close (fd) != 0)
    1173             :         {
    1174             :                 ELEKTRA_MMAP_LOG_WARNING ("could not close");
    1175             :         }
    1176             : 
    1177          12 :         keySetString (parentKey, keyString (initialParent));
    1178          12 :         if (initialParent) keyDel (initialParent);
    1179          12 :         errno = errnosave;
    1180          12 :         return ELEKTRA_PLUGIN_STATUS_ERROR;
    1181             : }
    1182             : 
    1183             : /**
    1184             :  * @brief The mmapstorage set function writes a keyset to a new file.
    1185             :  *
    1186             :  * The keyset is written to a new file. If the file was mapped previously,
    1187             :  * all current mappings are unlinked.
    1188             :  *
    1189             :  * @param handle The plugin handle.
    1190             :  * @param ks The keyset to be written.
    1191             :  * @param parentKey Holding the filename or error message.
    1192             :  *
    1193             :  * @retval ELEKTRA_PLUGIN_STATUS_SUCCESS if the file was written successfully.
    1194             :  * @retval ELEKTRA_PLUGIN_STATUS_ERROR if any error occurred.
    1195             :  */
    1196         346 : int ELEKTRA_PLUGIN_FUNCTION (set) (Plugin * handle ELEKTRA_UNUSED, KeySet * ks, Key * parentKey)
    1197             : {
    1198             :         // set all keys
    1199         346 :         KeySet * global = 0;
    1200         346 :         PluginMode mode = MODE_STORAGE;
    1201             : 
    1202         346 :         if ((global = elektraPluginGetGlobalKeySet (handle)) != 0)
    1203             :         {
    1204             :                 ELEKTRA_LOG_DEBUG ("mmapstorage global position called");
    1205          18 :                 mode = MODE_GLOBALCACHE;
    1206             :         }
    1207             : 
    1208         346 :         int errnosave = errno;
    1209         346 :         int fd = -1;
    1210         346 :         char * mappedRegion = MAP_FAILED;
    1211         346 :         DynArray * dynArray = 0;
    1212         346 :         Key * initialParent = keyDup (parentKey);
    1213             : 
    1214         346 :         if (elektraStrCmp (keyString (parentKey), STDOUT_FILENAME) == 0)
    1215             :         {
    1216             :                 // keySetString (parentKey, keyString (initialParent));
    1217           2 :                 fd = fileno (stdout);
    1218             :                 ELEKTRA_LOG_DEBUG ("MODE_NONREGULAR_FILE");
    1219           2 :                 set_bit (mode, MODE_NONREGULAR_FILE);
    1220             :         }
    1221             :         else
    1222             :         {
    1223         344 :                 if (unlink (keyString (parentKey)) != 0 && errno != ENOENT)
    1224             :                 {
    1225             :                         ELEKTRA_MMAP_LOG_WARNING ("could not unlink");
    1226             :                         goto error;
    1227             :                 }
    1228             : 
    1229         684 :                 if ((fd = openFile (parentKey, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR, mode)) == -1)
    1230             :                 {
    1231             :                         goto error;
    1232             :                 }
    1233             :         }
    1234             : 
    1235             :         struct stat sbuf;
    1236         344 :         if (fstatFile (fd, &sbuf, parentKey, mode) != 1)
    1237             :         {
    1238             :                 goto error;
    1239             :         }
    1240             : 
    1241         344 :         dynArray = ELEKTRA_PLUGIN_FUNCTION (dynArrayNew) ();
    1242             : 
    1243             :         MmapHeader mmapHeader;
    1244             :         MmapMetaData mmapMetaData;
    1245         344 :         initHeader (&mmapHeader);
    1246         344 :         initMetaData (&mmapMetaData);
    1247         344 :         calculateMmapDataSize (&mmapHeader, &mmapMetaData, ks, global, dynArray);
    1248             :         ELEKTRA_LOG_DEBUG ("mmapsize: %" PRIu64, mmapHeader.allocSize);
    1249             : 
    1250         686 :         if (!test_bit (mode, MODE_NONREGULAR_FILE) && truncateFile (fd, mmapHeader.allocSize, parentKey, mode) != 1)
    1251             :         {
    1252             :                 goto error;
    1253             :         }
    1254             : 
    1255         344 :         mappedRegion = mmapFile ((void *) 0, fd, mmapHeader.allocSize, MAP_SHARED, parentKey, mode);
    1256             :         ELEKTRA_LOG_DEBUG ("mappedRegion ptr: %p", (void *) mappedRegion);
    1257         344 :         if (mappedRegion == MAP_FAILED)
    1258             :         {
    1259             :                 ELEKTRA_MMAP_LOG_WARNING ("mappedRegion == MAP_FAILED");
    1260             :                 goto error;
    1261             :         }
    1262             : 
    1263             :         MmapFooter mmapFooter;
    1264         344 :         initFooter (&mmapFooter);
    1265         344 :         copyKeySetToMmap (mappedRegion, ks, global, &mmapHeader, &mmapMetaData, &mmapFooter, dynArray);
    1266             : 
    1267         344 :         if (test_bit (mode, MODE_NONREGULAR_FILE))
    1268             :         {
    1269           2 :                 ssize_t writtenBytes = write (fd, mappedRegion, mmapHeader.allocSize);
    1270           2 :                 if (writtenBytes == -1)
    1271             :                 {
    1272             :                         ELEKTRA_MMAP_LOG_WARNING ("could not write buffer to file");
    1273             :                         goto error;
    1274             :                 }
    1275           2 :                 elektraFree (mappedRegion);
    1276           2 :                 mappedRegion = MAP_FAILED;
    1277             :         }
    1278             : 
    1279         344 :         if (!test_bit (mode, MODE_NONREGULAR_FILE) && msync ((void *) mappedRegion, mmapHeader.allocSize, MS_SYNC) != 0)
    1280             :         {
    1281             :                 ELEKTRA_MMAP_LOG_WARNING ("could not msync");
    1282             :                 goto error;
    1283             :         }
    1284             : 
    1285         344 :         if (!test_bit (mode, MODE_NONREGULAR_FILE) && munmap (mappedRegion, mmapHeader.allocSize) != 0)
    1286             :         {
    1287             :                 ELEKTRA_MMAP_LOG_WARNING ("could not munmap");
    1288             :                 goto error;
    1289             :         }
    1290             : 
    1291         344 :         if (close (fd) != 0)
    1292             :         {
    1293             :                 ELEKTRA_MMAP_LOG_WARNING ("could not close");
    1294             :                 goto error;
    1295             :         }
    1296             : 
    1297         344 :         ELEKTRA_PLUGIN_FUNCTION (dynArrayDelete) (dynArray);
    1298         344 :         keySetString (parentKey, keyString (initialParent));
    1299         344 :         if (initialParent) keyDel (initialParent);
    1300             :         return ELEKTRA_PLUGIN_STATUS_SUCCESS;
    1301             : 
    1302             : error:
    1303           2 :         if (errno != 0)
    1304             :         {
    1305             :                 ELEKTRA_MMAP_LOG_WARNING ("strerror: %s", strerror (errno));
    1306           2 :                 ELEKTRA_SET_ERROR_SET (parentKey);
    1307             :         }
    1308             : 
    1309           2 :         if ((mappedRegion != MAP_FAILED && !test_bit (mode, MODE_NONREGULAR_FILE)) && (munmap (mappedRegion, mmapHeader.allocSize) != 0))
    1310             :         {
    1311             :                 ELEKTRA_MMAP_LOG_WARNING ("could not munmap");
    1312             :         }
    1313             : 
    1314           2 :         if (mappedRegion != MAP_FAILED && test_bit (mode, MODE_NONREGULAR_FILE))
    1315             :         {
    1316           0 :                 elektraFree (mappedRegion);
    1317             :         }
    1318             : 
    1319           2 :         if ((fd != -1) && close (fd) != 0)
    1320             :         {
    1321             :                 ELEKTRA_MMAP_LOG_WARNING ("could not close");
    1322             :         }
    1323             : 
    1324           2 :         keySetString (parentKey, keyString (initialParent));
    1325           2 :         if (initialParent) keyDel (initialParent);
    1326           2 :         ELEKTRA_PLUGIN_FUNCTION (dynArrayDelete) (dynArray);
    1327             : 
    1328           2 :         errno = errnosave;
    1329           2 :         return ELEKTRA_PLUGIN_STATUS_ERROR;
    1330             : }
    1331             : 
    1332         608 : Plugin * ELEKTRA_PLUGIN_EXPORT
    1333             : {
    1334             :         // clang-format off
    1335         608 :         return elektraPluginExport (ELEKTRA_PLUGIN_NAME,
    1336             :                 ELEKTRA_PLUGIN_OPEN,    &ELEKTRA_PLUGIN_FUNCTION(open),
    1337             :                 ELEKTRA_PLUGIN_CLOSE,   &ELEKTRA_PLUGIN_FUNCTION(close),
    1338             :                 ELEKTRA_PLUGIN_GET,     &ELEKTRA_PLUGIN_FUNCTION(get),
    1339             :                 ELEKTRA_PLUGIN_SET,     &ELEKTRA_PLUGIN_FUNCTION(set),
    1340             :                 ELEKTRA_PLUGIN_END);
    1341             : }

Generated by: LCOV version 1.13