Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */ 2 : #ifndef _MM_PERCPU_INTERNAL_H 3 : #define _MM_PERCPU_INTERNAL_H 4 : 5 : #include <linux/types.h> 6 : #include <linux/percpu.h> 7 : 8 : /* 9 : * pcpu_block_md is the metadata block struct. 10 : * Each chunk's bitmap is split into a number of full blocks. 11 : * All units are in terms of bits. 12 : * 13 : * The scan hint is the largest known contiguous area before the contig hint. 14 : * It is not necessarily the actual largest contig hint though. There is an 15 : * invariant that the scan_hint_start > contig_hint_start iff 16 : * scan_hint == contig_hint. This is necessary because when scanning forward, 17 : * we don't know if a new contig hint would be better than the current one. 18 : */ 19 : struct pcpu_block_md { 20 : int scan_hint; /* scan hint for block */ 21 : int scan_hint_start; /* block relative starting 22 : position of the scan hint */ 23 : int contig_hint; /* contig hint for block */ 24 : int contig_hint_start; /* block relative starting 25 : position of the contig hint */ 26 : int left_free; /* size of free space along 27 : the left side of the block */ 28 : int right_free; /* size of free space along 29 : the right side of the block */ 30 : int first_free; /* block position of first free */ 31 : int nr_bits; /* total bits responsible for */ 32 : }; 33 : 34 : struct pcpu_chunk { 35 : #ifdef CONFIG_PERCPU_STATS 36 : int nr_alloc; /* # of allocations */ 37 : size_t max_alloc_size; /* largest allocation size */ 38 : #endif 39 : 40 : struct list_head list; /* linked to pcpu_slot lists */ 41 : int free_bytes; /* free bytes in the chunk */ 42 : struct pcpu_block_md chunk_md; 43 : void *base_addr; /* base address of this chunk */ 44 : 45 : unsigned long *alloc_map; /* allocation map */ 46 : unsigned long *bound_map; /* boundary map */ 47 : struct pcpu_block_md *md_blocks; /* metadata blocks */ 48 : 49 : void *data; /* chunk data */ 50 : bool immutable; /* no [de]population allowed */ 51 : bool isolated; /* isolated from active chunk 52 : slots */ 53 : int start_offset; /* the overlap with the previous 54 : region to have a page aligned 55 : base_addr */ 56 : int end_offset; /* additional area required to 57 : have the region end page 58 : aligned */ 59 : #ifdef CONFIG_MEMCG_KMEM 60 : struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 61 : #endif 62 : 63 : int nr_pages; /* # of pages served by this chunk */ 64 : int nr_populated; /* # of populated pages */ 65 : int nr_empty_pop_pages; /* # of empty populated pages */ 66 : unsigned long populated[]; /* populated bitmap */ 67 : }; 68 : 69 : extern spinlock_t pcpu_lock; 70 : 71 : extern struct list_head *pcpu_chunk_lists; 72 : extern int pcpu_nr_slots; 73 : extern int pcpu_sidelined_slot; 74 : extern int pcpu_to_depopulate_slot; 75 : extern int pcpu_nr_empty_pop_pages; 76 : 77 : extern struct pcpu_chunk *pcpu_first_chunk; 78 : extern struct pcpu_chunk *pcpu_reserved_chunk; 79 : 80 : /** 81 : * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 82 : * @chunk: chunk of interest 83 : * 84 : * This conversion is from the number of physical pages that the chunk 85 : * serves to the number of bitmap blocks used. 86 : */ 87 : static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 88 : { 89 1575 : return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 90 : } 91 : 92 : /** 93 : * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 94 : * @pages: number of physical pages 95 : * 96 : * This conversion is from physical pages to the number of bits 97 : * required in the bitmap. 98 : */ 99 : static inline int pcpu_nr_pages_to_map_bits(int pages) 100 : { 101 1143 : return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 102 : } 103 : 104 : /** 105 : * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 106 : * @chunk: chunk of interest 107 : * 108 : * This conversion is from the number of physical pages that the chunk 109 : * serves to the number of bits in the bitmap. 110 : */ 111 : static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 112 : { 113 2286 : return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 114 : } 115 : 116 : #ifdef CONFIG_MEMCG_KMEM 117 : /** 118 : * pcpu_obj_full_size - helper to calculate size of each accounted object 119 : * @size: size of area to allocate in bytes 120 : * 121 : * For each accounted object there is an extra space which is used to store 122 : * obj_cgroup membership. Charge it too. 123 : */ 124 : static inline size_t pcpu_obj_full_size(size_t size) 125 : { 126 : size_t extra_size; 127 : 128 : extra_size = size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *); 129 : 130 : return size * num_possible_cpus() + extra_size; 131 : } 132 : #endif /* CONFIG_MEMCG_KMEM */ 133 : 134 : #ifdef CONFIG_PERCPU_STATS 135 : 136 : #include <linux/spinlock.h> 137 : 138 : struct percpu_stats { 139 : u64 nr_alloc; /* lifetime # of allocations */ 140 : u64 nr_dealloc; /* lifetime # of deallocations */ 141 : u64 nr_cur_alloc; /* current # of allocations */ 142 : u64 nr_max_alloc; /* max # of live allocations */ 143 : u32 nr_chunks; /* current # of live chunks */ 144 : u32 nr_max_chunks; /* max # of live chunks */ 145 : size_t min_alloc_size; /* min allocation size */ 146 : size_t max_alloc_size; /* max allocation size */ 147 : }; 148 : 149 : extern struct percpu_stats pcpu_stats; 150 : extern struct pcpu_alloc_info pcpu_stats_ai; 151 : 152 : /* 153 : * For debug purposes. We don't care about the flexible array. 154 : */ 155 : static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 156 : { 157 : memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 158 : 159 : /* initialize min_alloc_size to unit_size */ 160 : pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 161 : } 162 : 163 : /* 164 : * pcpu_stats_area_alloc - increment area allocation stats 165 : * @chunk: the location of the area being allocated 166 : * @size: size of area to allocate in bytes 167 : * 168 : * CONTEXT: 169 : * pcpu_lock. 170 : */ 171 : static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 172 : { 173 : lockdep_assert_held(&pcpu_lock); 174 : 175 : pcpu_stats.nr_alloc++; 176 : pcpu_stats.nr_cur_alloc++; 177 : pcpu_stats.nr_max_alloc = 178 : max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 179 : pcpu_stats.min_alloc_size = 180 : min(pcpu_stats.min_alloc_size, size); 181 : pcpu_stats.max_alloc_size = 182 : max(pcpu_stats.max_alloc_size, size); 183 : 184 : chunk->nr_alloc++; 185 : chunk->max_alloc_size = max(chunk->max_alloc_size, size); 186 : } 187 : 188 : /* 189 : * pcpu_stats_area_dealloc - decrement allocation stats 190 : * @chunk: the location of the area being deallocated 191 : * 192 : * CONTEXT: 193 : * pcpu_lock. 194 : */ 195 : static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 196 : { 197 : lockdep_assert_held(&pcpu_lock); 198 : 199 : pcpu_stats.nr_dealloc++; 200 : pcpu_stats.nr_cur_alloc--; 201 : 202 : chunk->nr_alloc--; 203 : } 204 : 205 : /* 206 : * pcpu_stats_chunk_alloc - increment chunk stats 207 : */ 208 : static inline void pcpu_stats_chunk_alloc(void) 209 : { 210 : unsigned long flags; 211 : spin_lock_irqsave(&pcpu_lock, flags); 212 : 213 : pcpu_stats.nr_chunks++; 214 : pcpu_stats.nr_max_chunks = 215 : max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 216 : 217 : spin_unlock_irqrestore(&pcpu_lock, flags); 218 : } 219 : 220 : /* 221 : * pcpu_stats_chunk_dealloc - decrement chunk stats 222 : */ 223 : static inline void pcpu_stats_chunk_dealloc(void) 224 : { 225 : unsigned long flags; 226 : spin_lock_irqsave(&pcpu_lock, flags); 227 : 228 : pcpu_stats.nr_chunks--; 229 : 230 : spin_unlock_irqrestore(&pcpu_lock, flags); 231 : } 232 : 233 : #else 234 : 235 : static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 236 : { 237 : } 238 : 239 : static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 240 : { 241 : } 242 : 243 : static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 244 : { 245 : } 246 : 247 : static inline void pcpu_stats_chunk_alloc(void) 248 : { 249 : } 250 : 251 : static inline void pcpu_stats_chunk_dealloc(void) 252 : { 253 : } 254 : 255 : #endif /* !CONFIG_PERCPU_STATS */ 256 : 257 : #endif
