33#ifndef TSAN_ANNOTATIONS
34#define TSAN_ANNOTATIONS 0
39const unsigned __tsan_mutex_linker_init = 1 << 0;
40void __tsan_mutex_pre_lock(
void *addr,
unsigned flags);
41void __tsan_mutex_post_lock(
void *addr,
unsigned flags,
int recursion);
42int __tsan_mutex_pre_unlock(
void *addr,
unsigned flags);
43void __tsan_mutex_post_unlock(
void *addr,
unsigned flags);
44void __tsan_mutex_pre_signal(
void *addr,
unsigned flags);
45void __tsan_mutex_post_signal(
void *addr,
unsigned flags);
53namespace Synchronization {
101 if (spin_count > 0) {
104 return spin_count > 0;
113static constexpr uint8_t lock_bit = 0x01;
114static constexpr uint8_t queue_lock_bit = 0x02;
115static constexpr uint8_t parked_bit = 0x02;
178WEAK void word_lock::lock_full() {
199 word_lock_queue_data node;
201 node.parker.prepare_park();
204 word_lock_queue_data *head = (word_lock_queue_data *)(
expected & ~(
uintptr_t)(queue_lock_bit | lock_bit));
205 if (head ==
nullptr) {
225WEAK void word_lock::unlock_full() {
246 word_lock_queue_data *head = (word_lock_queue_data *)(
expected & ~(
uintptr_t)(queue_lock_bit | lock_bit));
247 word_lock_queue_data *
current = head;
248 word_lock_queue_data *tail =
current->tail;
249 while (tail ==
nullptr) {
250 word_lock_queue_data *next =
current->next;
269 word_lock_queue_data *
new_tail = tail->prev;
296 tail->parker.unpark_start();
297 tail->parker.unpark();
298 tail->parker.unpark_finish();
335 while (head !=
nullptr) {
410 if (&buckets.
from == &buckets.
to) {
412 }
else if (&buckets.
from > &buckets.
to) {
455 return action.invalid_unpark_info;
461 if (
bucket.head !=
nullptr) {
484 while (data !=
nullptr) {
493 if (
bucket.tail == data) {
507 data->
parker.unpark_start();
510 data->
parker.unpark_finish();
545 while (data !=
nullptr) {
580 if (buckets.
to.
head ==
nullptr) {
591 wakeup->unpark_info = unpark_info;
592 wakeup->parker.unpark_start();
595 wakeup->parker.unpark_finish();
614 return result == (lock_bit | parked_bit);
708 if (!(val & lock_bit)) {
903 fast_cond->
wait(fast_mutex);
916 if (array ==
nullptr) {
922 if (array->
array ==
nullptr) {
This file declares the routines used by Halide internally in its runtime.
void * halide_malloc(void *user_context, size_t x)
Halide calls these functions to allocate and free memory.
void halide_free(void *user_context, void *ptr)
@ halide_error_code_success
There was no error.
ALWAYS_INLINE void signal()
ALWAYS_INLINE void broadcast()
ALWAYS_INLINE void wait(fast_mutex *mutex)
ALWAYS_INLINE void make_parked()
ALWAYS_INLINE void lock()
ALWAYS_INLINE void unlock()
ALWAYS_INLINE bool make_parked_if_locked()
ALWAYS_INLINE bool should_spin()
ALWAYS_INLINE void reset()
ALWAYS_INLINE void unlock()
ALWAYS_INLINE void lock()
WEAK hash_bucket & lock_bucket(uintptr_t addr)
WEAK void unlock_bucket_pair(bucket_pair &buckets)
constexpr int HASH_TABLE_SIZE
constexpr int HASH_TABLE_BITS
WEAK bucket_pair lock_bucket_pair(uintptr_t addr_from, uintptr_t addr_to)
constexpr int LOAD_FACTOR
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Internal::ConstantInterval cast(Type t, const Internal::ConstantInterval &a)
Cast operators for ConstantIntervals.
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
unsigned __INT64_TYPE__ uint64_t
#define halide_debug_assert(user_context, cond)
halide_debug_assert() is like halide_assert(), but only expands into a check when DEBUG_RUNTIME is de...
__UINTPTR_TYPE__ uintptr_t
unsigned __INT8_TYPE__ uint8_t
void halide_thread_yield()
void * memset(void *s, int val, size_t n)
#define halide_abort_if_false(user_context, cond)
uintptr_t *const cond_state
ALWAYS_INLINE broadcast_parking_control(uintptr_t *cond_state, fast_mutex *mutex)
bool validate(validate_action &action) final
void requeue_callback(const validate_action &action, bool one_to_wake, bool some_requeued) final
ALWAYS_INLINE bucket_pair(hash_bucket &from, hash_bucket &to)
hash_bucket buckets[HASH_TABLE_SIZE]
uintptr_t *const lock_state
bool validate(validate_action &action) final
uintptr_t unpark(int unparked, bool more_waiters) final
ALWAYS_INLINE mutex_parking_control(uintptr_t *lock_state)
virtual uintptr_t unpark(int unparked, bool more_waiters)
virtual void requeue_callback(const validate_action &action, bool one_to_wake, bool some_requeued)
uintptr_t park(uintptr_t addr)
int unpark_requeue(uintptr_t addr_from, uintptr_t addr_to, uintptr_t unpark_info)
virtual void before_sleep()
uintptr_t unpark_one(uintptr_t addr)
virtual bool validate(validate_action &action)
ALWAYS_INLINE signal_parking_control(uintptr_t *cond_state, fast_mutex *mutex)
uintptr_t unpark(int unparked, bool more_waiters) final
uintptr_t *const cond_state
uintptr_t invalid_unpark_info
uintptr_t unpark(int unparked, bool more_waiters) final
ALWAYS_INLINE wait_parking_control(uintptr_t *cond_state, fast_mutex *mutex)
uintptr_t *const cond_state
void before_sleep() final
bool validate(validate_action &action) final
word_lock_queue_data * prev
word_lock_queue_data * tail
word_lock_queue_data * next
Cross platform condition variable.
struct halide_mutex * array
WEAK int halide_mutex_array_lock(struct halide_mutex_array *array, int entry)
WEAK void halide_mutex_array_destroy(void *user_context, void *array)
WEAK void halide_mutex_unlock(halide_mutex *mutex)
WEAK void halide_cond_signal(struct halide_cond *cond)
WEAK int halide_mutex_array_unlock(struct halide_mutex_array *array, int entry)
WEAK void halide_mutex_lock(halide_mutex *mutex)
A basic set of mutex and condition variable functions, which call platform specific code for mutual e...
WEAK void halide_cond_wait(struct halide_cond *cond, struct halide_mutex *mutex)
WEAK void halide_cond_broadcast(struct halide_cond *cond)
WEAK halide_mutex_array * halide_mutex_array_create(uint64_t sz)