剖析 stl + glibc “内存泄漏” 原因

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
/* g++ -g -std='c++11' example_pressure.cpp -o ep111  && ./ep111 10000 */
 
#include <string.h>
#include <unistd.h>
 
#include <iostream>
#include <list>
 
int main(int argc, char** argv) {
    if (argc != 2) {
        printf("./proc [count]\n");
        return -1;
    }
 
    int cnt = atoi(argv[1]);
    std::list<char*> free_list;
 
    for (int i = 0; i < cnt; i++) {
        char* m = new char[1024 * 64];
        memset(m, 'a', 1024 * 64);
        free_list.push_back(m);
    }
 
    for (auto& v : free_list) {
        delete[] v;
    }
 
    for (;;) {
        sleep(1);
    }
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
/* example_pressure.cpp */
...
void print_mem_info(const char* s) {
    printf("------------------------\n");
    printf("-- %s\n", s);
    printf("------------------------\n");
    malloc_stats();
}

int main(int argc, char** argv) {
    ...
    int cnt = atoi(argv[1]);
    std::list<char*> free_list;

    print_mem_info("begin");

    for (int i = 0; i < cnt; i++) {
        char* m = new char[1024 * 64];
        memset(m, 'a', 1024 * 64);
        free_list.push_back(m);
    }

    print_mem_info("alloc blocks");

    for (auto& v : free_list) {
        delete[] v;
    }

    print_mem_info("free blocks");

    free_list.clear();
    print_mem_info("clear list.");
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/*
  This struct declaration is misleading (but accurate and necessary).
  It declares a "view" into memory allowing access to necessary
  fields at known offsets from a given base. See explanation below.
*/
struct malloc_chunk {
    INTERNAL_SIZE_T prev_size; /* Size of previous chunk (if free).  */
    INTERNAL_SIZE_T size;      /* Size in bytes, including overhead. */

    struct malloc_chunk* fd; /* double links -- used only if free. */
    struct malloc_chunk* bk;

    /* Only used for large blocks: pointer to next larger size.  */
    struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
    struct malloc_chunk* bk_nextsize;
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
typedef struct malloc_chunk* mchunkptr;
typedef struct malloc_chunk* mfastbinptr;

struct malloc_state {
    ...
    mfastbinptr fastbinsY[NFASTBINS];

    /* Base of the topmost chunk -- not otherwise kept in a bin */
    mchunkptr top;

    /* Normal bins packed as described above */
    mchunkptr bins[NBINS * 2 - 2];
    ...
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
void print_mem_info(const char* s) {
    printf("------------------------\n");
    printf("-- %s\n", s);
    printf("------------------------\n");
    malloc_stats();
    printf("------------------------\n");
}

void test() {
    char *addr, *addr2;
    int size = 64 * 1024;
    char* mems[64 * 1024];

    for (int i = 0; i < size; i++) {
        addr = (char*)malloc(4 * 1024 * sizeof(char));
        mems[i] = addr;
    }

    /* 紧贴 top chunk 的小内存块 addr2，如果一直不释放，那它前面分配的内存即使 free 掉后，
       内存池依然不会将缓存返还给系统。 */
    addr2 = (char*)malloc(1 * sizeof(char));

    for (int i = 0; i < size; i++) {
        free(mems[i]);
    }

    print_mem_info("stats info");
    for (;;) {
        sleep(1);
    }
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
static void* _int_malloc(mstate av, size_t bytes) {
    ...
use_top:
    victim = av->top;
    size = chunksize(victim);

    /* 如果内存池没有匹配的缓存，那么从当前 top chunk 内存块划一块内存处理分配。 */
    if ((unsigned long)(size) >= (unsigned long)(nb + MINSIZE)) {
        remainder_size = size - nb;
        remainder = chunk_at_offset(victim, nb);
        av->top = remainder;
        set_head(victim, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0));
        set_head(remainder, remainder_size | PREV_INUSE);

        check_malloced_chunk(av, victim, nb);
        void* p = chunk2mem(victim);
        if (__builtin_expect(perturb_byte, 0))
            alloc_perturb(p, bytes);
        return p;
    }
    /* 如果上面的 top chunk 没有足够的内存分配，那么考虑将 fast chunks 缓存的小块内存进行合并，
     *（在这里缓存的小内存块有可能前后都有比较大的空闲内存块，相互合并后，可能会合并出比较大的空闲空间。）
       看看是否能合并出足够的空间，提供分配。 */
    else if (have_fastchunks(av)) {
        malloc_consolidate(av);
        /* restore original bin index */
        if (in_smallbin_range(nb))
            idx = smallbin_index(nb);
        else
            idx = largebin_index(nb);
    }
    else {
        /* 经过上面两个步骤后，如果还是没有足够的内存，只有向内核申请内存。 */
        void* p = sysmalloc(nb, av);
        if (p != NULL && __builtin_expect(perturb_byte, 0))
            alloc_perturb(p, bytes);
        return p;
    }
    ...
}

#ifndef MORECORE
#define MORECORE sbrk
#endif

static void* sysmalloc(INTERNAL_SIZE_T nb, mstate av) {
    ...
    if (contiguous(av))
        size -= old_size;
    size = (size + pagemask) & ~pagemask;
    ...
    if (size > 0)
        brk = (char*)(MORECORE(size));
    ...
    /* Cannot merge with old top, so add its size back in */
    if (contiguous(av))
        size = (size + old_size + pagemask) & ~pagemask;
    ...
    /* finally, do the allocation */
    p = av->top;
    size = chunksize(p);

    /* 经过 sbrk 内存分配，top chunk 已经有足够的内存空间了，那么从它那里将 nb 大小的内存划分出去。 */
    if ((unsigned long)(size) >= (unsigned long)(nb + MINSIZE)) {
        remainder_size = size - nb;
        remainder = chunk_at_offset(p, nb);
        av->top = remainder;
        set_head(p, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0));
        set_head(remainder, remainder_size | PREV_INUSE);
        check_malloced_chunk(av, p, nb);
        return chunk2mem(p);
    }
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#ifndef DEFAULT_MXFAST
#define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
#endif

#define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL)

static void _int_free(mstate av, mchunkptr p, int have_lock) {
    ...
    /* 小内存块有可能被缓存起来，例如 std::list 32 个字节的节点。 */
    if ((unsigned long)(size) <= (unsigned long)(get_max_fast()) /* DEFAULT_MXFAST */
#if TRIM_FASTBINS
        /*
            If TRIM_FASTBINS set, don't place chunks
            bordering top into fastbins
       */
        && (chunk_at_offset(p, size) != av->top)
#endif
    ) {
        /* 小内存块缓存到 fastbin. */
        ...
    }
    ...
    else if (!chunk_is_mmapped(p)) {
        /* free 掉的内存，可能很大，也可能因为前后有空闲块，合并成一个大空闲块，
         * top chunk 也可能达到释放的阈值，尝试收缩返还系统。 */
        if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
            if (have_fastchunks(av))
                malloc_consolidate(av);

            if (av == &main_arena) {
#ifndef MORECORE_CANNOT_TRIM
                if ((unsigned long)(chunksize(av->top)) >= (unsigned long)(mp_.trim_threshold))
                    systrim(mp_.top_pad, av);
#endif
            }
            ...
        }
        ...
    }
}

#ifndef MORECORE
#define MORECORE sbrk
#endif

static int systrim(size_t pad, mstate av) {
    ...
    pagesz = GLRO(dl_pagesize);
    top_size = chunksize(av->top);

    /* 释放 top chunk 这个块超出阈值那部分（以页为单位）。 */
    extra = (top_size - pad - MINSIZE - 1) & ~(pagesz - 1);

    if (extra > 0) {
        current_brk = (char*)(MORECORE(0));
        if (current_brk == (char*)(av->top) + top_size) {
            /* 以页为单位回收 top chunk 多出来的内存。 */
            MORECORE(-extra);
            ...
        }
    }
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
/*
  ------------------------------ malloc_trim ------------------------------
*/
static int mtrim(mstate av, size_t pad) {
    /* 整理合并 fastbin 缓存的空闲小内存块。 */
    malloc_consolidate(av);

    const size_t ps = GLRO(dl_pagesize);
    int psindex = bin_index(ps);
    const size_t psm1 = ps - 1;

    int result = 0;
    for (int i = 1; i < NBINS; ++i)
        if (i == 1 || i >= psindex) {
            mbinptr bin = bin_at(av, i);

            for (mchunkptr p = last(bin); p != bin; p = p->bk) {
                INTERNAL_SIZE_T size = chunksize(p);

                if (size > psm1 + sizeof(struct malloc_chunk)) {
                    /* See whether the chunk contains at least one unused page.  */
                    char* paligned_mem = (char*)(((uintptr_t)p + sizeof(struct malloc_chunk) + psm1) & ~psm1);

                    assert((char*)chunk2mem(p) + 4 * SIZE_SZ <= paligned_mem);
                    assert((char*)p + size > paligned_mem);

                    /* This is the size we could potentially free.  */
                    size -= paligned_mem - (char*)p;

                    if (size > psm1) {
#ifdef MALLOC_DEBUG
                        /* When debugging we simulate destroying the memory content.  */
                        memset(paligned_mem, 0x89, size & ~psm1);
#endif
                        /* 回收达到一定数值的空闲内存块，将其从物理内存清除。 */
                        __madvise(paligned_mem, size & ~psm1, MADV_DONTNEED);

                        result = 1;
                    }
                }
            }
        }

#ifndef MORECORE_CANNOT_TRIM
    return result | (av == &main_arena ? systrim(pad, av) : 0);
#else
    return result;
#endif
}

static void malloc_consolidate(mstate av) {
    ...
    if (get_max_fast() != 0) {
        clear_fastchunks(av);
        unsorted_bin = unsorted_chunks(av);
        maxfb = &fastbin(av, NFASTBINS - 1);
        fb = &fastbin(av, 0);
        do {
            p = atomic_exchange_acq(fb, 0);
            if (p != 0) {
                do {
                    ...
                    if (nextchunk != av->top) {
                        ...
                    } else {
                        /* top chunk 有可能会因为空闲小内存的回收合并而增加，超过返还系统的阈值。 */
                        size += nextsize;
                        set_head(p, size | PREV_INUSE);
                        av->top = p;
                    }

                } while ((p = nextp) != 0);
            }
        } while (fb++ != maxfb);
    }
    ...
}