Training courses

Kernel and Embedded Linux

Bootlin training courses

Embedded Linux, kernel,
Yocto Project, Buildroot, real-time,
graphics, boot time, debugging...

Bootlin logo

Elixir Cross Referencer

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
//===-- tsan_mman.cpp -----------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is a part of ThreadSanitizer (TSan), a race detector.
//
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_allocator_report.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "tsan_mman.h"
#include "tsan_rtl.h"
#include "tsan_report.h"
#include "tsan_flags.h"

// May be overriden by front-end.
SANITIZER_WEAK_DEFAULT_IMPL
void __sanitizer_malloc_hook(void *ptr, uptr size) {
  (void)ptr;
  (void)size;
}

SANITIZER_WEAK_DEFAULT_IMPL
void __sanitizer_free_hook(void *ptr) {
  (void)ptr;
}

namespace __tsan {

struct MapUnmapCallback {
  void OnMap(uptr p, uptr size) const { }
  void OnUnmap(uptr p, uptr size) const {
    // We are about to unmap a chunk of user memory.
    // Mark the corresponding shadow memory as not needed.
    DontNeedShadowFor(p, size);
    // Mark the corresponding meta shadow memory as not needed.
    // Note the block does not contain any meta info at this point
    // (this happens after free).
    const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
    const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
    // Block came from LargeMmapAllocator, so must be large.
    // We rely on this in the calculations below.
    CHECK_GE(size, 2 * kPageSize);
    uptr diff = RoundUp(p, kPageSize) - p;
    if (diff != 0) {
      p += diff;
      size -= diff;
    }
    diff = p + size - RoundDown(p + size, kPageSize);
    if (diff != 0)
      size -= diff;
    uptr p_meta = (uptr)MemToMeta(p);
    ReleaseMemoryPagesToOS(p_meta, p_meta + size / kMetaRatio);
  }
};

static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
Allocator *allocator() {
  return reinterpret_cast<Allocator*>(&allocator_placeholder);
}

struct GlobalProc {
  Mutex mtx;
  Processor *proc;

  GlobalProc()
      : mtx(MutexTypeGlobalProc, StatMtxGlobalProc)
      , proc(ProcCreate()) {
  }
};

static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
GlobalProc *global_proc() {
  return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
}

ScopedGlobalProcessor::ScopedGlobalProcessor() {
  GlobalProc *gp = global_proc();
  ThreadState *thr = cur_thread();
  if (thr->proc())
    return;
  // If we don't have a proc, use the global one.
  // There are currently only two known case where this path is triggered:
  //   __interceptor_free
  //   __nptl_deallocate_tsd
  //   start_thread
  //   clone
  // and:
  //   ResetRange
  //   __interceptor_munmap
  //   __deallocate_stack
  //   start_thread
  //   clone
  // Ideally, we destroy thread state (and unwire proc) when a thread actually
  // exits (i.e. when we join/wait it). Then we would not need the global proc
  gp->mtx.Lock();
  ProcWire(gp->proc, thr);
}

ScopedGlobalProcessor::~ScopedGlobalProcessor() {
  GlobalProc *gp = global_proc();
  ThreadState *thr = cur_thread();
  if (thr->proc() != gp->proc)
    return;
  ProcUnwire(gp->proc, thr);
  gp->mtx.Unlock();
}

static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
static uptr max_user_defined_malloc_size;

void InitializeAllocator() {
  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
  allocator()->Init(common_flags()->allocator_release_to_os_interval_ms);
  max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
                                     ? common_flags()->max_allocation_size_mb
                                           << 20
                                     : kMaxAllowedMallocSize;
}

void InitializeAllocatorLate() {
  new(global_proc()) GlobalProc();
}

void AllocatorProcStart(Processor *proc) {
  allocator()->InitCache(&proc->alloc_cache);
  internal_allocator()->InitCache(&proc->internal_alloc_cache);
}

void AllocatorProcFinish(Processor *proc) {
  allocator()->DestroyCache(&proc->alloc_cache);
  internal_allocator()->DestroyCache(&proc->internal_alloc_cache);
}

void AllocatorPrintStats() {
  allocator()->PrintStats();
}

static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
  if (atomic_load_relaxed(&thr->in_signal_handler) == 0 ||
      !flags()->report_signal_unsafe)
    return;
  VarSizeStackTrace stack;
  ObtainCurrentStack(thr, pc, &stack);
  if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
    return;
  ThreadRegistryLock l(ctx->thread_registry);
  ScopedReport rep(ReportTypeSignalUnsafe);
  rep.AddStack(stack, true);
  OutputReport(thr, rep);
}


void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
                          bool signal) {
  if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize ||
      sz > max_user_defined_malloc_size) {
    if (AllocatorMayReturnNull())
      return nullptr;
    uptr malloc_limit =
        Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
  }
  void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
  if (UNLIKELY(!p)) {
    SetAllocatorOutOfMemory();
    if (AllocatorMayReturnNull())
      return nullptr;
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportOutOfMemory(sz, &stack);
  }
  if (ctx && ctx->initialized)
    OnUserAlloc(thr, pc, (uptr)p, sz, true);
  if (signal)
    SignalUnsafeCall(thr, pc);
  return p;
}

void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
  ScopedGlobalProcessor sgp;
  if (ctx && ctx->initialized)
    OnUserFree(thr, pc, (uptr)p, true);
  allocator()->Deallocate(&thr->proc()->alloc_cache, p);
  if (signal)
    SignalUnsafeCall(thr, pc);
}

void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment));
}

void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
    if (AllocatorMayReturnNull())
      return SetErrnoOnNull(nullptr);
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportCallocOverflow(n, size, &stack);
  }
  void *p = user_alloc_internal(thr, pc, n * size);
  if (p)
    internal_memset(p, 0, n * size);
  return SetErrnoOnNull(p);
}

void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) {
  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
    if (AllocatorMayReturnNull())
      return SetErrnoOnNull(nullptr);
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportReallocArrayOverflow(size, n, &stack);
  }
  return user_realloc(thr, pc, p, size * n);
}

void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
  DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
  ctx->metamap.AllocBlock(thr, pc, p, sz);
  if (write && thr->ignore_reads_and_writes == 0)
    MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
  else
    MemoryResetRange(thr, pc, (uptr)p, sz);
}

void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
  CHECK_NE(p, (void*)0);
  uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);
  DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz);
  if (write && thr->ignore_reads_and_writes == 0)
    MemoryRangeFreed(thr, pc, (uptr)p, sz);
}

void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
  // FIXME: Handle "shrinking" more efficiently,
  // it seems that some software actually does this.
  if (!p)
    return SetErrnoOnNull(user_alloc_internal(thr, pc, sz));
  if (!sz) {
    user_free(thr, pc, p);
    return nullptr;
  }
  void *new_p = user_alloc_internal(thr, pc, sz);
  if (new_p) {
    uptr old_sz = user_alloc_usable_size(p);
    internal_memcpy(new_p, p, min(old_sz, sz));
    user_free(thr, pc, p);
  }
  return SetErrnoOnNull(new_p);
}

void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
  if (UNLIKELY(!IsPowerOfTwo(align))) {
    errno = errno_EINVAL;
    if (AllocatorMayReturnNull())
      return nullptr;
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportInvalidAllocationAlignment(align, &stack);
  }
  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
}

int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
                        uptr sz) {
  if (UNLIKELY(!CheckPosixMemalignAlignment(align))) {
    if (AllocatorMayReturnNull())
      return errno_EINVAL;
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportInvalidPosixMemalignAlignment(align, &stack);
  }
  void *ptr = user_alloc_internal(thr, pc, sz, align);
  if (UNLIKELY(!ptr))
    // OOM error is already taken care of by user_alloc_internal.
    return errno_ENOMEM;
  CHECK(IsAligned((uptr)ptr, align));
  *memptr = ptr;
  return 0;
}

void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) {
  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) {
    errno = errno_EINVAL;
    if (AllocatorMayReturnNull())
      return nullptr;
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportInvalidAlignedAllocAlignment(sz, align, &stack);
  }
  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
}

void *user_valloc(ThreadState *thr, uptr pc, uptr sz) {
  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached()));
}

void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
  uptr PageSize = GetPageSizeCached();
  if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) {
    errno = errno_ENOMEM;
    if (AllocatorMayReturnNull())
      return nullptr;
    GET_STACK_TRACE_FATAL(thr, pc);
    ReportPvallocOverflow(sz, &stack);
  }
  // pvalloc(0) should allocate one page.
  sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
}

uptr user_alloc_usable_size(const void *p) {
  if (p == 0)
    return 0;
  MBlock *b = ctx->metamap.GetBlock((uptr)p);
  if (!b)
    return 0;  // Not a valid pointer.
  if (b->siz == 0)
    return 1;  // Zero-sized allocations are actually 1 byte.
  return b->siz;
}

void invoke_malloc_hook(void *ptr, uptr size) {
  ThreadState *thr = cur_thread();
  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
    return;
  __sanitizer_malloc_hook(ptr, size);
  RunMallocHooks(ptr, size);
}

void invoke_free_hook(void *ptr) {
  ThreadState *thr = cur_thread();
  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
    return;
  __sanitizer_free_hook(ptr);
  RunFreeHooks(ptr);
}

void *internal_alloc(MBlockType typ, uptr sz) {
  ThreadState *thr = cur_thread();
  if (thr->nomalloc) {
    thr->nomalloc = 0;  // CHECK calls internal_malloc().
    CHECK(0);
  }
  return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
}

void internal_free(void *p) {
  ThreadState *thr = cur_thread();
  if (thr->nomalloc) {
    thr->nomalloc = 0;  // CHECK calls internal_malloc().
    CHECK(0);
  }
  InternalFree(p, &thr->proc()->internal_alloc_cache);
}

}  // namespace __tsan

using namespace __tsan;

extern "C" {
uptr __sanitizer_get_current_allocated_bytes() {
  uptr stats[AllocatorStatCount];
  allocator()->GetStats(stats);
  return stats[AllocatorStatAllocated];
}

uptr __sanitizer_get_heap_size() {
  uptr stats[AllocatorStatCount];
  allocator()->GetStats(stats);
  return stats[AllocatorStatMapped];
}

uptr __sanitizer_get_free_bytes() {
  return 1;
}

uptr __sanitizer_get_unmapped_bytes() {
  return 1;
}

uptr __sanitizer_get_estimated_allocated_size(uptr size) {
  return size;
}

int __sanitizer_get_ownership(const void *p) {
  return allocator()->GetBlockBegin(p) != 0;
}

uptr __sanitizer_get_allocated_size(const void *p) {
  return user_alloc_usable_size(p);
}

void __tsan_on_thread_idle() {
  ThreadState *thr = cur_thread();
  thr->clock.ResetCached(&thr->proc()->clock_cache);
  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
  allocator()->SwallowCache(&thr->proc()->alloc_cache);
  internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
  ctx->metamap.OnProcIdle(thr->proc());
}
}  // extern "C"