unicodeUtils.h

Go to the documentation of this file.

//
// Copyright 2023 Pixar
//
// Licensed under the Apache License, Version 2.0 (the "Apache License")
// with the following modification; you may not use this file except in
// compliance with the Apache License and the following modification to it:
// Section 6. Trademarks. is deleted and replaced with:
//
// 6. Trademarks. This License does not grant permission to use the trade
//    names, trademarks, service marks, or product names of the Licensor
//    and its affiliates, except as required to comply with Section 4(c) of
//    the License and to reproduce the content of the NOTICE file.
//
// You may obtain a copy of the Apache License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the Apache License with the above modification is
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the Apache License for the specific
// language governing permissions and limitations under the Apache License.
//
#ifndef PXR_BASE_TF_UNICODE_UTILS_H
#define PXR_BASE_TF_UNICODE_UTILS_H
 
 
#include "pxr/pxr.h"
#include "pxr/base/tf/api.h"
#include "pxr/base/tf/diagnostic.h"
 
#include <ostream>
#include <string>
#include <string_view>
 
PXR_NAMESPACE_OPEN_SCOPE
 
class TfUtf8CodePoint {
public:
    static constexpr uint32_t ReplacementValue = 0xFFFD;
 
    static constexpr uint32_t MaximumValue = 0x10FFFF;
 
    static constexpr std::pair<uint32_t, uint32_t>
    SurrogateRange = {0xD800, 0xDFFF};
 
    constexpr TfUtf8CodePoint() = default;
 
    constexpr explicit TfUtf8CodePoint(uint32_t value) :
        _value(((value <= MaximumValue) &&
                ((value < SurrogateRange.first) ||
                 (value > SurrogateRange.second))) ?
               value : ReplacementValue) {}
 
    constexpr uint32_t AsUInt32() const { return _value; }
 
    friend constexpr bool operator==(const TfUtf8CodePoint left,
                                     const TfUtf8CodePoint right) {
        return left._value == right._value;
    }
    friend constexpr bool operator!=(const TfUtf8CodePoint left,
                                     const TfUtf8CodePoint right) {
        return left._value != right._value;
    }
 
private:
    uint32_t _value{ReplacementValue};
};
 
TF_API std::ostream& operator<<(std::ostream&, const TfUtf8CodePoint);
 
constexpr TfUtf8CodePoint TfUtf8InvalidCodePoint{
    TfUtf8CodePoint::ReplacementValue};
 
constexpr TfUtf8CodePoint TfUtf8CodePointFromAscii(const char value)
{
    return static_cast<unsigned char>(value) < 128 ?
           TfUtf8CodePoint(static_cast<unsigned char>(value)) :
           TfUtf8InvalidCodePoint;
}
 
class TfUtf8CodePointIterator final {
public:
    using iterator_category = std::forward_iterator_tag;
    using value_type = TfUtf8CodePoint;
    using difference_type = std::ptrdiff_t;
    using pointer = void;
    using reference = TfUtf8CodePoint;
 
    class PastTheEndSentinel final {};
 
    TfUtf8CodePointIterator(
        const std::string_view::const_iterator& it,
        const std::string_view::const_iterator& end) : _it(it), _end(end) {
            TF_DEV_AXIOM(_it <= _end);
        }
 
    value_type operator* () const
    {
        return TfUtf8CodePoint{_GetCodePoint()};
    }
 
    std::string_view::const_iterator GetBase() const
    {
        return this->_it;
    }
 
    bool operator== (const TfUtf8CodePointIterator& rhs) const
    {
        return (this->_it == rhs._it);
    }
 
    bool operator!= (const TfUtf8CodePointIterator& rhs) const
    {
        return (this->_it != rhs._it);
    }
 
    TfUtf8CodePointIterator& operator++ ()
    {
        // The increment operator should never be called if it's past
        // the end. The user is expected to have already checked this
        // condition.
        TF_DEV_AXIOM(!_IsPastTheEnd());
        _EncodingLength increment = _GetEncodingLength();
        // Note that in cases where the encoding is invalid, we move to the
        // next byte. This is necessary because otherwise the iterator would
        // never advance and the end condition of == iterator::end() would
        // never be satisfied. This means that we increment, even if the
        // encoding length is 0.
        ++_it;
        // Only continuation bytes will be consumed after the the first byte.
        // This avoids consumption of ASCII characters or other starting bytes.
        auto isContinuation = [](const char c) {
            const auto uc = static_cast<unsigned char>(c);
            return (uc >= static_cast<unsigned char>('\x80')) &&
                   (uc < static_cast<unsigned char>('\xc0'));
        };
        while ((increment > 1) && !_IsPastTheEnd() && isContinuation(*_it)) {
            ++_it;
            --increment;
        }
        return *this;
    }
 
    TfUtf8CodePointIterator operator++ (int)
    {
        auto temp = *this;
        ++(*this);
        return temp;
    }
 
    friend bool operator==(const TfUtf8CodePointIterator& lhs,
                           PastTheEndSentinel)
    {
        return lhs._IsPastTheEnd();
    }
 
    friend bool operator==(PastTheEndSentinel lhs,
                           const TfUtf8CodePointIterator& rhs)
    {
        return rhs == lhs;
    }
 
    friend bool operator!=(const TfUtf8CodePointIterator& lhs,
                           PastTheEndSentinel rhs)
    {
        return !(lhs == rhs);
    }
    friend bool operator!=(PastTheEndSentinel lhs,
                           const TfUtf8CodePointIterator& rhs)
    {
        return !(lhs == rhs);
    }
 
private:
    using _EncodingLength = unsigned char;
 
    // Retrieves the variable encoding length of the UTF-8 character
    // currently pointed to by the iterator. This can be 1, 2, 3, or 4
    // depending on the encoding of the UTF-8 character. If the encoding
    // cannot be determined, this method will return 0.
    _EncodingLength _GetEncodingLength() const
    {
        // already at the end, no valid character sequence
        if (_IsPastTheEnd())
        {
            return 0;
        }
        // determine what encoding length the character is
        // 1-byte characters have a leading 0 sequence
        // 2-byte characters have a leading 110 sequence
        // 3-byte characters have a leading 1110 sequence
        // 4-byte characters have a leading 11110 sequence
        unsigned char x = static_cast<unsigned char>(*_it);
        if (x < 0x80)
        {
            return 1;
        }
        else if ((x >= 0xc0) && (x < 0xe0))
        {
            return 2;
        }
        else if ((x >= 0xe0) && (x < 0xf0))
        {
            return 3;
        }
        else if ((x >= 0xf0) && (x < 0xf8))
        {
            return 4;
        }
        else
        {
            // can't determine encoding, this is an error
            return 0;
        }
    }
 
    // Retrieves the Unicode code point of the next character in the UTF-8
    // encoded sequence (defined by \a begin) and returns the value in
    // \a codePoint. This method will return \a true if the encoded
    // sequence is valid. If the encoding is invalid, this method will
    // return \a false and \a codePoint will be set to 0.
    TF_API uint32_t _GetCodePoint() const;
 
    // Returns true if the iterator at or past the end and can no longer be
    // dereferenced.
    bool _IsPastTheEnd() const
    {
        return _it >= _end;
    }
 
    std::string_view::const_iterator _it;
    std::string_view::const_iterator _end;
};
 
class TfUtf8CodePointView final {
public:
    using const_iterator = TfUtf8CodePointIterator;
 
    TfUtf8CodePointView() = default;
    explicit TfUtf8CodePointView(const std::string_view& view) : _view(view) {}
 
    inline const_iterator begin() const
    {
        return const_iterator{std::cbegin(_view), std::cend(_view)};
    }
 
    TfUtf8CodePointIterator::PastTheEndSentinel end() const
    {
        return TfUtf8CodePointIterator::PastTheEndSentinel{};
    }
 
    inline const_iterator cbegin() const
    {
        return begin();
    }
 
    TfUtf8CodePointIterator::PastTheEndSentinel cend() const
    {
        return end();
    }
 
    bool empty() const
    {
        return _view.empty();
    }
 
    inline const_iterator EndAsIterator() const
    {
        return const_iterator(std::cend(_view), std::cend(_view));
    }
 
private:
    std::string_view _view;
};
 
TF_API
bool TfIsUtf8CodePointXidStart(uint32_t codePoint);
 
inline bool TfIsUtf8CodePointXidStart(const TfUtf8CodePoint codePoint)
{
    return TfIsUtf8CodePointXidStart(codePoint.AsUInt32());
}
 
TF_API
bool TfIsUtf8CodePointXidContinue(uint32_t codePoint);
 
inline bool TfIsUtf8CodePointXidContinue(const TfUtf8CodePoint codePoint)
{
    return TfIsUtf8CodePointXidContinue(codePoint.AsUInt32());
}
 
PXR_NAMESPACE_CLOSE_SCOPE
 
#endif // PXR_BASE_TF_UNICODE_UTILS_H_