lock_core.h

Go to the documentation of this file.

/*
 * Copyright (c) 2020 Raspberry Pi (Trading) Ltd.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */
 
#ifndef _PICO_LOCK_CORE_H
#define _PICO_LOCK_CORE_H
 
#include "pico.h"
#include "pico/time.h"
#include "hardware/sync.h"

 
// PICO_CONFIG: PARAM_ASSERTIONS_ENABLED_LOCK_CORE, Enable/disable assertions in the lock core, type=bool, default=0, group=pico_sync
#ifndef PARAM_ASSERTIONS_ENABLED_LOCK_CORE
#define PARAM_ASSERTIONS_ENABLED_LOCK_CORE 0
#endif
 
// PICO_CONFIG: PICO_SYNC_EXCLUSIVE_ACCESS_EVENT_WORKAROUND, Enable workaround to preserve low power waits in synchronization primitives where an exclusive access sets the calling core's own event, type=bool, default=1 when using software spin locks on such a platform, advanced=true, group=pico_sync
#ifndef PICO_SYNC_EXCLUSIVE_ACCESS_EVENT_WORKAROUND
#ifdef PICO_SYNC_RP2350_SPIN_LOCK_WORKAROUND
#define PICO_SYNC_EXCLUSIVE_ACCESS_EVENT_WORKAROUND PICO_SYNC_RP2350_SPIN_LOCK_WORKAROUND
#else
#define PICO_SYNC_EXCLUSIVE_ACCESS_EVENT_WORKAROUND (PICO_USE_SW_SPIN_LOCKS && PICO_EXCLUSIVE_ACCESS_SETS_OWN_EVENT)
#endif
#endif


struct lock_core {
    spin_lock_t *spin_lock; 
 
    // note any lock members in containing structures need not be volatile;
    // they are protected by memory/compiler barriers when gaining and release spin locks
};
 
typedef struct lock_core lock_core_t;

void lock_init(lock_core_t *core, uint lock_num);
 
#ifndef lock_owner_id_t
#define lock_owner_id_t int8_t
#endif
 
#ifndef LOCK_INVALID_OWNER_ID
#define LOCK_INVALID_OWNER_ID ((lock_owner_id_t)-1)
#endif
 
#ifndef lock_get_caller_owner_id
#define lock_get_caller_owner_id() ((lock_owner_id_t)get_core_num())
#ifndef lock_is_owner_id_valid
#define lock_is_owner_id_valid(id) ((id)>=0)
#endif
#endif
 
#ifndef lock_is_owner_id_valid
#define lock_is_owner_id_valid(id) ((id) != LOCK_INVALID_OWNER_ID)
#endif
 
#ifdef lock_internal_spin_unlock_with_wait
#define LOCK_INTERNAL_SPIN_UNLOCK_WITH_WAIT_OVERRIDDEN 1
#else
#if !PICO_SYNC_EXCLUSIVE_ACCESS_EVENT_WORKAROUND
#define lock_internal_spin_unlock_with_wait(lock, save) spin_unlock((lock)->spin_lock, save), __wfe()
#else
extern volatile uint8_t lock_internal_notify_count;
// Note the ordering here matters. The event register is a single bit, with multiple sources: our
// own spin_unlock, an SEV from a notifier, an SEV from some other unrelated code, or (on Arm) an
// exception entry + return on this core. We must drain the event left by the spin_unlock *before*
// deciding whether we still need to wait for a second event from a notifier.
//
// The first (draining) __wfe() is inside the lock's IRQ critical section. Return-from-interrupt
// generates an event on Armv8-M, but not on Hazard3 v1.0, so IRQs can swallow events on RISC-V.
// This first __wfe() always completes promptly, because spin_unlock_unsafe() generates an event.
//
// With the drain first, a notify occurring after _notify_count is sampled either: (a) bumps the
// count before we test it, so we skip the second __wfe(), or (b) happens after the test, in which
// case its SEV sets the event register and the second __wfe() returns immediately.
//
// (Also note: the increment cannot happen between the _notify_count and the spin_unlock_unsafe()
// because the increment must hold the lock *we are initially holding*.)
//
// See comment on `lock_internal_notify_count` declaration in lock_core.c for background details
// on interactions between events and exclusives on RP2350.
#define lock_internal_spin_unlock_with_wait(lock, save) ({         \
    uint8_t _notify_count = lock_internal_notify_count;            \
    spin_unlock_unsafe((lock)->spin_lock);                         \
    __wfe(); /* consume event from unlock, without interruption */ \
    restore_interrupts_from_disabled(save);                        \
    if (_notify_count == lock_internal_notify_count) __wfe();      \
    })
#endif
#endif
 
#ifdef lock_internal_spin_unlock_with_notify
#define LOCK_INTERNAL_SPIN_UNLOCK_WITH_NOTIFY_OVERRIDDEN 1
#else
#if !PICO_SYNC_EXCLUSIVE_ACCESS_EVENT_WORKAROUND
#define lock_internal_spin_unlock_with_notify(lock, save) spin_unlock((lock)->spin_lock, save), __sev()
#else
// note that spin_lock_blocking() already posts an event to the current core: ldaexb/strex on Arm,
// amoor.w.aq on RISC-V, so creates + retires a reservation. (spin_unlock() doesn't as it's just a
// release-ordered store, but the point is a lock+unlock will always cause a core-local event.)
#define lock_internal_spin_unlock_with_notify(lock, save) ({ \
    lock_internal_notify_count++;                            \
    spin_unlock((lock)->spin_lock, save);                    \
    __sev();                                                 \
    })
#endif
#endif
 
#ifdef lock_internal_spin_unlock_with_best_effort_wait_or_timeout
#define LOCK_INTERNAL_SPIN_UNLOCK_WITH_BEST_EFFORT_WAIT_OR_TIMEOUT_OVERRIDDEN 1
#else
#if !PICO_SYNC_EXCLUSIVE_ACCESS_EVENT_WORKAROUND
#define lock_internal_spin_unlock_with_best_effort_wait_or_timeout(lock, save, until) ({ \
    spin_unlock((lock)->spin_lock, save);                                                \
    best_effort_wfe_or_timeout(until);                                                   \
})
#else
// see the comment on lock_internal_spin_unlock_with_wait above for why the event left by the
// spin_unlock must be drained before the notify count is tested
#define lock_internal_spin_unlock_with_best_effort_wait_or_timeout(lock, save, until) ({ \
    uint8_t _notify_count = lock_internal_notify_count;                                  \
    spin_unlock_unsafe((lock)->spin_lock);                                               \
    __wfe(); /* consume event from spin_unlock_unsafe() without interruption */          \
    restore_interrupts_from_disabled(save);                                              \
    _notify_count == lock_internal_notify_count ? best_effort_wfe_or_timeout(until)      \
                                                : time_reached(until);                   \
})
#endif
#endif

#ifndef LOCK_INTERNAL_SPIN_UNLOCK_WITH_NOTIFY_WAKES_ALL
#if !(LOCK_INTERNAL_SPIN_UNLOCK_WITH_WAIT_OVERRIDDEN | LOCK_INTERNAL_SPIN_UNLOCK_WITH_NOTIFY_OVERRIDDEN | LOCK_INTERNAL_SPIN_UNLOCK_WITH_BEST_EFFORT_WAIT_OR_TIMEOUT_OVERRIDDEN)
#define LOCK_INTERNAL_SPIN_UNLOCK_WITH_NOTIFY_WAKES_ALL 1
#else
#define LOCK_INTERNAL_SPIN_UNLOCK_WITH_NOTIFY_WAKES_ALL 0
#endif
#endif

#if LOCK_INTERNAL_SPIN_UNLOCK_WITH_NOTIFY_WAKES_ALL
#define lock_internal_spin_unlock_maybe_notify(lock, save, others_may_proceed) spin_unlock((lock)->spin_lock, save)
#else
#define lock_internal_spin_unlock_maybe_notify(lock, save, others_may_proceed) ({ \
    if (others_may_proceed) {                                                     \
        lock_internal_spin_unlock_with_notify(lock, save);                        \
    } else {                                                                      \
        spin_unlock((lock)->spin_lock, save);                                     \
    }                                                                             \
})
#endif
 
#ifndef sync_internal_yield_until_before
#define sync_internal_yield_until_before(until) ((void)0)
#endif
 
#endif