Leptonica  1.82.0
Image processing and image analysis suite
pixalloc.c
Go to the documentation of this file.
1 /*====================================================================*
2  - Copyright (C) 2001 Leptonica. All rights reserved.
3  -
4  - Redistribution and use in source and binary forms, with or without
5  - modification, are permitted provided that the following conditions
6  - are met:
7  - 1. Redistributions of source code must retain the above copyright
8  - notice, this list of conditions and the following disclaimer.
9  - 2. Redistributions in binary form must reproduce the above
10  - copyright notice, this list of conditions and the following
11  - disclaimer in the documentation and/or other materials
12  - provided with the distribution.
13  -
14  - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
15  - ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
16  - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
17  - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ANY
18  - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21  - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
22  - OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
23  - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24  - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *====================================================================*/
26 
44 #ifdef HAVE_CONFIG_H
45 #include <config_auto.h>
46 #endif /* HAVE_CONFIG_H */
47 
48 #include "allheaders.h"
49 
50 /*-------------------------------------------------------------------------*
51  * Pix Memory Storage *
52  * *
53  * This is a simple utility for handling pix memory storage. It is *
54  * enabled by setting the PixMemoryManager allocators to the functions *
55  * that are defined here *
56  * pmsCustomAlloc() *
57  * pmsCustomDealloc() *
58  * Use pmsCreate() at the beginning to do the pre-allocation, and *
59  * pmsDestroy() at the end to clean it up. *
60  *-------------------------------------------------------------------------*/
61 /*
62  * In the following, the "memory" refers to the image data
63  * field that is used within the pix. The memory store is a
64  * continuous block of memory, that is logically divided into
65  * smaller "chunks" starting with a set at a minimum size, and
66  * followed by sets of increasing size that are a power of 2 larger
67  * than the minimum size. You must specify the number of chunks
68  * of each size.
69  *
70  * A requested data chunk, if it exists, is borrowed from the memory
71  * storage, and returned after use. If the chunk is too small, or
72  * too large, or if all chunks in the appropriate size range are
73  * in use, the memory is allocated dynamically and freed after use.
74  *
75  * There are four parameters that determine the use of pre-allocated memory:
76  *
77  * minsize: any requested chunk smaller than this is allocated
78  * dynamically and destroyed after use. No preallocated
79  * memory is used.
80  * smallest: the size of the smallest pre-allocated memory chunk.
81  * nlevels: the number of different sizes of data chunks, each a
82  * power of 2 larger than 'smallest'.
83  * numalloc: a Numa of size 'nlevels' containing the number of data
84  * chunks for each size that are in the memory store.
85  *
86  * As an example, suppose:
87  * minsize = 0.5MB
88  * smallest = 1.0MB
89  * nlevels = 4
90  * numalloc = {10, 5, 5, 5}
91  * Then the total amount of allocated memory (in MB) is
92  * 10 * 1 + 5 * 2 + 5 * 4 + 5 * 8 = 80 MB
93  * Any pix requiring less than 0.5 MB or more than 8 MB of memory will
94  * not come from the memory store. Instead, it will be dynamically
95  * allocated and freed after use.
96  *
97  * How is this implemented?
98  *
99  * At setup, the full data block size is computed and allocated.
100  * The addresses of the individual chunks are found, and the pointers
101  * are stored in a set of Ptra (generic pointer arrays), using one Ptra
102  * for each of the sizes of the chunks. When returning a chunk after
103  * use, it is necessary to determine from the address which size level
104  * (ptra) the chunk belongs to. This is done by comparing the address
105  * of the associated chunk.
106  *
107  * In the event that memory chunks need to be dynamically allocated,
108  * either (1) because they are too small or too large for the memory
109  * store or (2) because all the pix of that size (i.e., in the
110  * appropriate level) in the memory store are in use, the
111  * addresses generated will be outside the pre-allocated block.
112  * After use they won't be returned to a ptra; instead the deallocator
113  * will free them.
114  */
115 
118 {
119  struct L_Ptraa *paa;
120  size_t minsize;
122  size_t smallest;
123  size_t largest;
124  size_t nbytes;
125  l_int32 nlevels;
126  size_t *sizes;
127  l_int32 *allocarray;
128  l_uint32 *baseptr;
129  l_uint32 *maxptr;
130  l_uint32 **firstptr;
131  l_int32 *memused;
132  l_int32 *meminuse;
133  l_int32 *memmax;
134  l_int32 *memempty;
136  char *logfile;
137 };
138 typedef struct PixMemoryStore L_PIX_MEM_STORE;
139 
140 static L_PIX_MEM_STORE *CustomPMS = NULL;
141 
142 
171 l_ok
173  size_t smallest,
174  NUMA *numalloc,
175  const char *logfile)
176 {
177 l_int32 nlevels, i, j, nbytes;
178 l_int32 *alloca;
179 l_float32 nchunks;
180 l_uint32 *baseptr, *data;
181 l_uint32 **firstptr;
182 size_t *sizes;
183 L_PIX_MEM_STORE *pms;
184 L_PTRA *pa;
185 L_PTRAA *paa;
186 
187  PROCNAME("createPMS");
188 
189  if (!numalloc)
190  return ERROR_INT("numalloc not defined", procName, 1);
191  numaGetSum(numalloc, &nchunks);
192  if (nchunks > 1000.0)
193  L_WARNING("There are %.0f chunks\n", procName, nchunks);
194 
195  pms = (L_PIX_MEM_STORE *)LEPT_CALLOC(1, sizeof(L_PIX_MEM_STORE));
196  CustomPMS = pms;
197 
198  /* Make sure that minsize and smallest are multiples of 32 bit words */
199  if (minsize % 4 != 0)
200  minsize -= minsize % 4;
201  pms->minsize = minsize;
202  nlevels = numaGetCount(numalloc);
203  pms->nlevels = nlevels;
204 
205  if ((sizes = (size_t *)LEPT_CALLOC(nlevels, sizeof(size_t))) == NULL)
206  return ERROR_INT("sizes not made", procName, 1);
207  pms->sizes = sizes;
208  if (smallest % 4 != 0)
209  smallest += 4 - (smallest % 4);
210  pms->smallest = smallest;
211  for (i = 0; i < nlevels; i++)
212  sizes[i] = smallest * (1 << i);
213  pms->largest = sizes[nlevels - 1];
214 
215  alloca = numaGetIArray(numalloc);
216  pms->allocarray = alloca;
217  if ((paa = ptraaCreate(nlevels)) == NULL)
218  return ERROR_INT("paa not made", procName, 1);
219  pms->paa = paa;
220 
221  for (i = 0, nbytes = 0; i < nlevels; i++)
222  nbytes += alloca[i] * sizes[i];
223  pms->nbytes = nbytes;
224 
225  if ((baseptr = (l_uint32 *)LEPT_CALLOC(nbytes / 4, sizeof(l_uint32)))
226  == NULL)
227  return ERROR_INT("calloc fail for baseptr", procName, 1);
228  pms->baseptr = baseptr;
229  pms->maxptr = baseptr + nbytes / 4; /* just beyond the memory store */
230  if ((firstptr = (l_uint32 **)LEPT_CALLOC(nlevels, sizeof(l_uint32 *)))
231  == NULL)
232  return ERROR_INT("calloc fail for firstptr", procName, 1);
233  pms->firstptr = firstptr;
234 
235  data = baseptr;
236  for (i = 0; i < nlevels; i++) {
237  if ((pa = ptraCreate(alloca[i])) == NULL)
238  return ERROR_INT("pa not made", procName, 1);
239  ptraaInsertPtra(paa, i, pa);
240  firstptr[i] = data;
241  for (j = 0; j < alloca[i]; j++) {
242  ptraAdd(pa, data);
243  data += sizes[i] / 4;
244  }
245  }
246 
247  if (logfile) {
248  pms->memused = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
249  pms->meminuse = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
250  pms->memmax = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
251  pms->memempty = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
252  pms->logfile = stringNew(logfile);
253  }
254 
255  return 0;
256 }
257 
258 
268 void
270 {
271 L_PIX_MEM_STORE *pms;
272 
273  if ((pms = CustomPMS) == NULL)
274  return;
275 
276  ptraaDestroy(&pms->paa, FALSE, FALSE); /* don't touch the ptrs */
277  LEPT_FREE(pms->baseptr); /* free the memory */
278 
279  if (pms->logfile) {
280  pmsLogInfo();
281  LEPT_FREE(pms->logfile);
282  LEPT_FREE(pms->memused);
283  LEPT_FREE(pms->meminuse);
284  LEPT_FREE(pms->memmax);
285  LEPT_FREE(pms->memempty);
286  }
287 
288  LEPT_FREE(pms->sizes);
289  LEPT_FREE(pms->allocarray);
290  LEPT_FREE(pms->firstptr);
291  LEPT_FREE(pms);
292  CustomPMS = NULL;
293 }
294 
295 
311 void *
313 {
314 l_int32 level;
315 void *data;
316 L_PIX_MEM_STORE *pms;
317 L_PTRA *pa;
318 
319  PROCNAME("pmsCustomAlloc");
320 
321  if ((pms = CustomPMS) == NULL)
322  return (void *)ERROR_PTR("pms not defined", procName, NULL);
323 
324  pmsGetLevelForAlloc(nbytes, &level);
325 
326  if (level < 0) { /* size range invalid; must alloc */
327  if ((data = pmsGetAlloc(nbytes)) == NULL)
328  return (void *)ERROR_PTR("data not made", procName, NULL);
329  } else { /* get from store */
330  pa = ptraaGetPtra(pms->paa, level, L_HANDLE_ONLY);
331  data = ptraRemoveLast(pa);
332  if (data && pms->logfile) {
333  pms->memused[level]++;
334  pms->meminuse[level]++;
335  if (pms->meminuse[level] > pms->memmax[level])
336  pms->memmax[level]++;
337  }
338  if (!data) { /* none left at this level */
339  data = pmsGetAlloc(nbytes);
340  if (pms->logfile)
341  pms->memempty[level]++;
342  }
343  }
344 
345  return data;
346 }
347 
348 
355 void
356 pmsCustomDealloc(void *data)
357 {
358 l_int32 level;
359 L_PIX_MEM_STORE *pms;
360 L_PTRA *pa;
361 
362  PROCNAME("pmsCustomDealloc");
363 
364  if ((pms = CustomPMS) == NULL) {
365  L_ERROR("pms not defined\n", procName);
366  return;
367  }
368 
369  if (pmsGetLevelForDealloc(data, &level) == 1) {
370  L_ERROR("level not found\n", procName);
371  return;
372  }
373 
374  if (level < 0) { /* no logging; just free the data */
375  LEPT_FREE(data);
376  } else { /* return the data to the store */
377  pa = ptraaGetPtra(pms->paa, level, L_HANDLE_ONLY);
378  ptraAdd(pa, data);
379  if (pms->logfile)
380  pms->meminuse[level]--;
381  }
382 }
383 
384 
402 void *
404 {
405 void *data;
406 FILE *fp;
407 L_PIX_MEM_STORE *pms;
408 
409  PROCNAME("pmsGetAlloc");
410 
411  if ((pms = CustomPMS) == NULL)
412  return (void *)ERROR_PTR("pms not defined", procName, NULL);
413 
414  if ((data = (void *)LEPT_CALLOC(nbytes, sizeof(char))) == NULL)
415  return (void *)ERROR_PTR("data not made", procName, NULL);
416  if (pms->logfile && nbytes >= pms->smallest) {
417  if ((fp = fopenWriteStream(pms->logfile, "a")) != NULL) {
418  fprintf(fp, "Alloc %zu bytes at %p\n", nbytes, data);
419  fclose(fp);
420  } else {
421  L_ERROR("failed to open stream for %s\n", procName, pms->logfile);
422  }
423  }
424  return data;
425 }
426 
427 
435 l_ok
437  l_int32 *plevel)
438 {
439 l_int32 i;
440 l_float64 ratio;
441 L_PIX_MEM_STORE *pms;
442 
443  PROCNAME("pmsGetLevelForAlloc");
444 
445  if (!plevel)
446  return ERROR_INT("&level not defined", procName, 1);
447  *plevel = -1;
448  if ((pms = CustomPMS) == NULL)
449  return ERROR_INT("pms not defined", procName, 1);
450 
451  if (nbytes < pms->minsize || nbytes > pms->largest)
452  return 0; /* -1 */
453 
454  ratio = (l_float64)nbytes / (l_float64)(pms->smallest);
455  for (i = 0; i < pms->nlevels; i++) {
456  if (ratio <= 1.0)
457  break;
458  ratio /= 2.0;
459  }
460  *plevel = i;
461 
462  return 0;
463 }
464 
465 
474 l_ok
476  l_int32 *plevel)
477 {
478 l_int32 i;
479 l_uint32 *first;
480 L_PIX_MEM_STORE *pms;
481 
482  PROCNAME("pmsGetLevelForDealloc");
483 
484  if (!plevel)
485  return ERROR_INT("&level not defined", procName, 1);
486  *plevel = -1;
487  if (!data)
488  return ERROR_INT("data not defined", procName, 1);
489  if ((pms = CustomPMS) == NULL)
490  return ERROR_INT("pms not defined", procName, 1);
491 
492  if (data < (void *)pms->baseptr || data >= (void *)pms->maxptr)
493  return 0; /* -1 */
494 
495  for (i = 1; i < pms->nlevels; i++) {
496  first = pms->firstptr[i];
497  if (data < (void *)first)
498  break;
499  }
500  *plevel = i - 1;
501 
502  return 0;
503 }
504 
505 
509 void
511 {
512 l_int32 i;
513 L_PIX_MEM_STORE *pms;
514 
515  if ((pms = CustomPMS) == NULL)
516  return;
517 
518  lept_stderr("Total number of pix used at each level\n");
519  for (i = 0; i < pms->nlevels; i++)
520  lept_stderr(" Level %d (%zu bytes): %d\n", i,
521  pms->sizes[i], pms->memused[i]);
522 
523  lept_stderr("Max number of pix in use at any time in each level\n");
524  for (i = 0; i < pms->nlevels; i++)
525  lept_stderr(" Level %d (%zu bytes): %d\n", i,
526  pms->sizes[i], pms->memmax[i]);
527 
528  lept_stderr("Number of pix alloc'd because none were available\n");
529  for (i = 0; i < pms->nlevels; i++)
530  lept_stderr(" Level %d (%zu bytes): %d\n", i,
531  pms->sizes[i], pms->memempty[i]);
532 }
l_ok numaGetSum(NUMA *na, l_float32 *psum)
numaGetSum()
Definition: numafunc1.c:538
void pmsDestroy(void)
pmsDestroy()
Definition: pixalloc.c:269
size_t minsize
Definition: pixalloc.c:120
char * logfile
Definition: pixalloc.c:136
void ptraaDestroy(L_PTRAA **ppaa, l_int32 freeflag, l_int32 warnflag)
ptraaDestroy()
Definition: ptra.c:834
size_t smallest
Definition: pixalloc.c:122
char * stringNew(const char *src)
stringNew()
Definition: utils2.c:223
l_uint32 * maxptr
Definition: pixalloc.c:129
void pmsLogInfo(void)
pmsLogInfo()
Definition: pixalloc.c:510
size_t nbytes
Definition: pixalloc.c:124
void * pmsCustomAlloc(size_t nbytes)
pmsCustomAlloc()
Definition: pixalloc.c:312
void lept_stderr(const char *fmt,...)
lept_stderr()
Definition: utils1.c:306
void pmsCustomDealloc(void *data)
pmsCustomDealloc()
Definition: pixalloc.c:356
size_t * sizes
Definition: pixalloc.c:126
void * ptraRemoveLast(L_PTRA *pa)
ptraRemoveLast()
Definition: ptra.c:491
l_ok pmsGetLevelForAlloc(size_t nbytes, l_int32 *plevel)
pmsGetLevelForAlloc()
Definition: pixalloc.c:436
l_int32 * numaGetIArray(NUMA *na)
numaGetIArray()
Definition: numabasic.c:847
Definition: array.h:70
L_PTRA * ptraCreate(l_int32 n)
ptraCreate()
Definition: ptra.c:144
l_int32 numaGetCount(NUMA *na)
numaGetCount()
Definition: numabasic.c:658
L_PTRA * ptraaGetPtra(L_PTRAA *paa, l_int32 index, l_int32 accessflag)
ptraaGetPtra()
Definition: ptra.c:948
l_ok ptraaInsertPtra(L_PTRAA *paa, l_int32 index, L_PTRA *pa)
ptraaInsertPtra()
Definition: ptra.c:905
l_int32 nlevels
Definition: pixalloc.c:125
l_int32 * memused
Definition: pixalloc.c:131
Definition: ptra.h:53
l_int32 * allocarray
Definition: pixalloc.c:127
struct L_Ptraa * paa
Definition: pixalloc.c:119
l_ok pmsCreate(size_t minsize, size_t smallest, NUMA *numalloc, const char *logfile)
pmsCreate()
Definition: pixalloc.c:172
void * pmsGetAlloc(size_t nbytes)
pmsGetAlloc()
Definition: pixalloc.c:403
l_int32 * meminuse
Definition: pixalloc.c:132
l_uint32 ** firstptr
Definition: pixalloc.c:130
FILE * fopenWriteStream(const char *filename, const char *modestring)
fopenWriteStream()
Definition: utils2.c:1975
l_ok ptraAdd(L_PTRA *pa, void *item)
ptraAdd()
Definition: ptra.c:250
l_uint32 * baseptr
Definition: pixalloc.c:128
l_int32 * memempty
Definition: pixalloc.c:134
l_int32 * memmax
Definition: pixalloc.c:133
Definition: ptra.h:64
l_ok pmsGetLevelForDealloc(void *data, l_int32 *plevel)
pmsGetLevelForDealloc()
Definition: pixalloc.c:475
L_PTRAA * ptraaCreate(l_int32 n)
ptraaCreate()
Definition: ptra.c:798
size_t largest
Definition: pixalloc.c:123