// bsls_atomicoperations_arm_all_gcc.h                                -*-C++-*-
#ifndef INCLUDED_BSLS_ATOMICOPERATIONS_ARM_ALL_GCC
#define INCLUDED_BSLS_ATOMICOPERATIONS_ARM_ALL_GCC

#include <bsls_ident.h>
BSLS_IDENT("$Id: $")

//@PURPOSE: Provide implementations of atomic operations for ARM/GCC.
//
//@CLASSES:
//  bsls::AtomicOperations_ARM_ALL_GCC: implementation of atomics for ARM/GCC.
//
//@DESCRIPTION: This component provides classes necessary to implement atomics
// on the Linux ARM platform with GCC.  The classes are for private use only.
// See `bsls_atomicoperations` and `bsls_atomic` for the public interface to
// atomics.
//
// IMPLEMENTATION NOTES: Wherever possible we use atomic intrinsics that both
// GCC 4.6+ and Clang 3.2+ support.  Otherwise the inline assembly code is
// used that can be compiled both by GCC and Clang.  This puts some
// restrictions on the assembly code because Clang doesn't support register
// pairs that represent a 64bit value.  So instead of `ldrexd %1, %H1, [%3]`
// the exact registers have to be specified, as in `ldrexd r2, r3, [%3]`.  Note
// also that operations like `ldrexd` and `stdrexd` are supported only starting
// from `armv6zk`, so not every armv6 platform will work.
//
// For more details on ARM atomic primitives see the Linux kernel source code
// (arch/arm/include/asm/atomic.h) and the "C/C++11 mappings to processors"
// (http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html), and the
// "ARM1156T2F-S Technical Reference Manual"
// (http://infocenter.arm.com/help/index.jsp?
// topic=/com.arm.doc.ddi0290g/Babebdcb.html), which contains documentation on
// the p15 coprocessor opcodes used in this component).

#include <bsls_atomicoperations_default.h>
#include <bsls_platform.h>
#include <bsls_types.h>

#if defined(BSLS_PLATFORM_CPU_ARM) \
    && (defined(BSLS_PLATFORM_CMP_GNU) || defined(BSLS_PLATFORM_CMP_CLANG))

namespace BloombergLP {

namespace bsls {

struct AtomicOperations_ARM_ALL_GCC;
typedef AtomicOperations_ARM_ALL_GCC  AtomicOperations_Imp;

           // ======================================================
           // struct Atomic_TypeTraits<AtomicOperations_ARM_ALL_GCC>
           // ======================================================

template <>
struct Atomic_TypeTraits<AtomicOperations_ARM_ALL_GCC>
{
    struct Int
    {
        volatile int d_value __attribute__((__aligned__(sizeof(int))));
    };

    struct Int64
    {
        volatile Types::Int64 d_value
                       __attribute__((__aligned__(sizeof(Types::Int64))));
    };

    struct Uint
    {
        volatile unsigned int d_value
            __attribute__((__aligned__(sizeof(unsigned int))));
    };

    struct Uint64
    {
        volatile Types::Uint64 d_value
                       __attribute__((__aligned__(sizeof(Types::Uint64))));
    };

    struct Pointer
    {
        void * volatile d_value __attribute__((__aligned__(sizeof(void *))));
    };
};

                     // ===================================
                     // struct AtomicOperations_ARM_ALL_GCC
                     // ===================================

struct AtomicOperations_ARM_ALL_GCC
    : AtomicOperations_Default32<AtomicOperations_ARM_ALL_GCC>
{
private:
    // PRIVATE TYPES
    struct Int64_Words {
        int w1;
        int w2;
    };

public:
    typedef Atomic_TypeTraits<AtomicOperations_ARM_ALL_GCC> AtomicTypes;

        // *** atomic functions for int ***

    static int getInt(const AtomicTypes::Int *atomicInt);

    static int getIntAcquire(const AtomicTypes::Int *atomicInt);

    static void setInt(AtomicTypes::Int *atomicInt, int value);

    static void setIntRelease(AtomicTypes::Int *atomicInt, int value);

    static int swapInt(AtomicTypes::Int *atomicInt, int swapValue);

    static int testAndSwapInt(AtomicTypes::Int *atomicInt,
                              int compareValue,
                              int swapValue);

    static int addIntNv(AtomicTypes::Int *atomicInt, int value);

        // *** atomic functions for Int64 ***

    static Types::Int64 getInt64(const AtomicTypes::Int64 *atomicInt);

    static void setInt64(AtomicTypes::Int64 *atomicInt, Types::Int64 value);

    static Types::Int64 swapInt64(AtomicTypes::Int64  *atomicInt,
                                  Types::Int64 swapValue);

    static Types::Int64 testAndSwapInt64(AtomicTypes::Int64 *atomicInt,
                                         Types::Int64 compareValue,
                                         Types::Int64 swapValue);

    static Types::Int64 addInt64Nv(AtomicTypes::Int64 *atomicInt,
                                   Types::Int64 value);
};

// ===========================================================================
//                        INLINE FUNCTION DEFINITIONS
// ===========================================================================

                     // -----------------------------------
                     // struct AtomicOperations_ARM_ALL_GCC
                     // -----------------------------------

#if BSLS_PLATFORM_CMP_VERSION < 40700
#   if defined(BSLS_PLATFORM_CPU_ARM_V7)

#       define BSLS_ATOMICOPERATIONS_BARRIER()          \
            asm volatile ("dmb" ::: "memory")           \

#       define BSLS_ATOMICOPERATIONS_INSTR_BARRIER()    \
            asm volatile ("isb")                        \

#   else

#       define BSLS_ATOMICOPERATIONS_BARRIER()                      \
            do {                                                    \
                int temp_reg = 0;                                   \
                asm volatile (                                      \
                        "mcr p15, 0, %[temp_reg], c7, c10, 5 \n\t"  \
                        :                                           \
                        : [temp_reg] "r" (temp_reg)                 \
                        : "memory");                                \
            } while (0)                                             \

#       define BSLS_ATOMICOPERATIONS_INSTR_BARRIER()                \
            do {                                                    \
                int temp_reg = 0;                                   \
                asm volatile (                                      \
                        "mcr p15, 0, %[temp_reg], c7, c5, 4 \n\t"   \
                        :                                           \
                        : [temp_reg] "r" (temp_reg));               \
            } while (0)                                             \

#   endif
#else
#   define BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
#endif

inline
int AtomicOperations_ARM_ALL_GCC::
    getInt(const AtomicTypes::Int *atomicInt)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_load_n(&atomicInt->d_value, __ATOMIC_SEQ_CST);
#else
    int result = atomicInt->d_value;
    BSLS_ATOMICOPERATIONS_BARRIER();

    return result;

#endif
}

inline
int AtomicOperations_ARM_ALL_GCC::
    getIntAcquire(const AtomicTypes::Int *atomicInt)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_load_n(&atomicInt->d_value, __ATOMIC_ACQUIRE);
#else
    return getInt(atomicInt);
#endif
}

inline
void AtomicOperations_ARM_ALL_GCC::
    setInt(AtomicTypes::Int *atomicInt, int value)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    __atomic_store_n(&atomicInt->d_value, value, __ATOMIC_SEQ_CST);
#else
    BSLS_ATOMICOPERATIONS_BARRIER();
    atomicInt->d_value = value;
    BSLS_ATOMICOPERATIONS_BARRIER();
#endif
}

inline
void AtomicOperations_ARM_ALL_GCC::
    setIntRelease(AtomicTypes::Int *atomicInt, int value)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    __atomic_store_n(&atomicInt->d_value, value, __ATOMIC_RELEASE);
#else
    BSLS_ATOMICOPERATIONS_BARRIER();
    setInt(atomicInt, value);
#endif
}

inline
int AtomicOperations_ARM_ALL_GCC::
    swapInt(AtomicTypes::Int *atomicInt, int swapValue)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_exchange_n(&atomicInt->d_value,
                               swapValue,
                               __ATOMIC_SEQ_CST);
#else
    int oldValue;
    int tmp;

    BSLS_ATOMICOPERATIONS_BARRIER();

    asm volatile (
        "1:     ldrex   %[old], [%3]            \n\t"
        "       strex   %[tmp], %[val], [%3]    \n\t"
        "       teq     %[tmp], #0              \n\t"
        "       bne     1b                      \n\t"

                : [old] "=&r" (oldValue),
                  [tmp] "=&r" (tmp),
                        "+Qo" (*atomicInt)
                :       "r"   (atomicInt),
                  [val] "r"   (swapValue)
                : "cc", "memory");

    BSLS_ATOMICOPERATIONS_INSTR_BARRIER();

    return oldValue;
#endif
}

inline
int AtomicOperations_ARM_ALL_GCC::
    testAndSwapInt(AtomicTypes::Int *atomicInt,
                   int compareValue,
                   int swapValue)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    __atomic_compare_exchange_n(&atomicInt->d_value, &compareValue, swapValue,
                                false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
    return compareValue;
#else
    return __sync_val_compare_and_swap(&atomicInt->d_value,
                                       compareValue,
                                       swapValue);
#endif
}

inline
int AtomicOperations_ARM_ALL_GCC::
    addIntNv(AtomicTypes::Int *atomicInt, int value)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_add_fetch(&atomicInt->d_value, value, __ATOMIC_SEQ_CST);
#else
    return __sync_add_and_fetch(&atomicInt->d_value, value);
#endif
}

inline
Types::Int64 AtomicOperations_ARM_ALL_GCC::
    getInt64(const AtomicTypes::Int64 *atomicInt)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_load_n(&atomicInt->d_value, __ATOMIC_SEQ_CST);
#else
    Types::Int64 result;

    asm volatile (
        "       ldrexd  r2, r3, [%2]       \n\t"
        "       mov     %[res1], r2        \n\t"
        "       mov     %[res2], r3        \n\t"

                : [res1] "=&r" (((Int64_Words &) result).w1),
                  [res2] "=&r" (((Int64_Words &) result).w2)
                :        "r"   (atomicInt),
                         "Qo"  (*atomicInt)
                : "r2", "r3");

    BSLS_ATOMICOPERATIONS_BARRIER();

    return result;
#endif
}

inline
void AtomicOperations_ARM_ALL_GCC::
    setInt64(AtomicTypes::Int64 *atomicInt, Types::Int64 value)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_store_n(&atomicInt->d_value, value, __ATOMIC_SEQ_CST);
#else
    swapInt64(atomicInt, value);
#endif
}

inline
Types::Int64 AtomicOperations_ARM_ALL_GCC::
    swapInt64(AtomicTypes::Int64 *atomicInt,
              Types::Int64 swapValue)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_exchange_n(&atomicInt->d_value,
                               swapValue,
                               __ATOMIC_SEQ_CST);
#else
    Types::Int64 oldValue;
    int rc;

    BSLS_ATOMICOPERATIONS_BARRIER();

    asm volatile (
        "1:     ldrexd  r2, r3, [%4]           \n\t"
        "       mov     %[old1], r2            \n\t"
        "       mov     %[old2], r3            \n\t"
        "       mov     r2, %[val1]            \n\t"
        "       mov     r3, %[val2]            \n\t"
        "       strexd  %[rc], r2, r3, [%4]    \n\t"
        "       teq     %[rc], #0              \n\t"
        "       bne     1b                     \n\t"

                : [rc]   "=&r" (rc),
                  [old1] "=&r" (((Int64_Words &) oldValue).w1),
                  [old2] "=&r" (((Int64_Words &) oldValue).w2),
                         "+Qo" (*atomicInt)
                :        "r"   (atomicInt),
                  [val1] "r"   (((Int64_Words &) swapValue).w1),
                  [val2] "r"   (((Int64_Words &) swapValue).w2)
                : "r2", "r3", "cc", "memory");

    BSLS_ATOMICOPERATIONS_INSTR_BARRIER();

    return oldValue;
#endif
}

inline
Types::Int64 AtomicOperations_ARM_ALL_GCC::
    testAndSwapInt64(AtomicTypes::Int64 *atomicInt,
                     Types::Int64 compareValue,
                     Types::Int64 swapValue)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    __atomic_compare_exchange_n(&atomicInt->d_value,
                                &compareValue,
                                swapValue,
                                false,
                                __ATOMIC_SEQ_CST,
                                __ATOMIC_SEQ_CST);
    return compareValue;
#else
    Types::Int64 oldValue;
    int rc;

    BSLS_ATOMICOPERATIONS_BARRIER();

    asm volatile (
        "1:     ldrexd  r2, r3, [%4]           \n\t"
        "       mov     %[old1], r2            \n\t"
        "       mov     %[old2], r3            \n\t"
        "       mov     r2, %[val1]            \n\t"
        "       mov     r3, %[val2]            \n\t"
        "       mov     %[rc], #0              \n\t"
        "       teq     %[old1], %[cmp1]       \n\t"
        "       itt     eq                     \n\t"
        "       teqeq   %[old2], %[cmp2]       \n\t"
        "       strexdeq %[rc], r2, r3, [%4]   \n\t"
        "       teq     %[rc], #0              \n\t"
        "       bne     1b                     \n\t"

                : [rc]   "=&r" (rc),
                  [old1] "=&r" (((Int64_Words &) oldValue).w1),
                  [old2] "=&r" (((Int64_Words &) oldValue).w2),
                         "+Qo" (*atomicInt)
                :        "r"   (atomicInt),
                  [cmp1] "r"   (((Int64_Words &) compareValue).w1),
                  [cmp2] "r"   (((Int64_Words &) compareValue).w2),
                  [val1] "r"   (((Int64_Words &) swapValue).w1),
                  [val2] "r"   (((Int64_Words &) swapValue).w2)
                : "r2", "r3", "cc", "memory");

    BSLS_ATOMICOPERATIONS_INSTR_BARRIER();

    return oldValue;
#endif
}

inline
Types::Int64 AtomicOperations_ARM_ALL_GCC::
    addInt64Nv(AtomicTypes::Int64 *atomicInt,
               Types::Int64 value)
{
#ifdef BSLS_ATOMICOPERATIONS_USE_CPP11_INTRINSICS
    return __atomic_add_fetch(&atomicInt->d_value, value, __ATOMIC_SEQ_CST);
#else
    Types::Int64 old;
    Types::Int64 newVal;
    Types::Int64 prev = atomicInt->d_value;

    do {
        old = prev;
        newVal = old + value;
    } while (old != (prev = testAndSwapInt64(atomicInt, old, newVal)));

    return newVal;
#endif
}

}  // close package namespace

}  // close enterprise namespace

#endif  // defined(BSLS_PLATFORM_CPU_ARM) && (CMP_GNU || CMP_CLANG)

#endif

// ----------------------------------------------------------------------------
// Copyright 2013 Bloomberg Finance L.P.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------- END-OF-FILE ----------------------------------
