- * We assume that the bitset data is word-aligned (that is, a multiple of 8 bytes in length). - *
- * Each bit occupies exactly one bit of storage. - */ -public final class BitSetMethods { - - private static final long WORD_SIZE = 8; - - private BitSetMethods() { - // Make the default constructor private, since this only holds static methods. - } - - /** - * Sets the bit at the specified index to {@code true}. - */ - public static void set(Object baseObject, long baseOffset, int index) { - assert index >= 0 : "index (" + index + ") should >= 0"; - final long mask = 1L << (index & 0x3f); // mod 64 and shift - final long wordOffset = baseOffset + (index >> 6) * WORD_SIZE; - final long word = Platform.getLong(baseObject, wordOffset); - Platform.putLong(baseObject, wordOffset, word | mask); - } - - /** - * Sets the bit at the specified index to {@code false}. - */ - public static void unset(Object baseObject, long baseOffset, int index) { - assert index >= 0 : "index (" + index + ") should >= 0"; - final long mask = 1L << (index & 0x3f); // mod 64 and shift - final long wordOffset = baseOffset + (index >> 6) * WORD_SIZE; - final long word = Platform.getLong(baseObject, wordOffset); - Platform.putLong(baseObject, wordOffset, word & ~mask); - } - - /** - * Returns {@code true} if the bit is set at the specified index. - */ - public static boolean isSet(Object baseObject, long baseOffset, int index) { - assert index >= 0 : "index (" + index + ") should >= 0"; - final long mask = 1L << (index & 0x3f); // mod 64 and shift - final long wordOffset = baseOffset + (index >> 6) * WORD_SIZE; - final long word = Platform.getLong(baseObject, wordOffset); - return (word & mask) != 0; - } - - /** - * Returns {@code true} if any bit is set. - */ - public static boolean anySet(Object baseObject, long baseOffset, long bitSetWidthInWords) { - long address = baseOffset; - for (int i = 0; i < bitSetWidthInWords; i++, address += WORD_SIZE) { - if (Platform.getLong(baseObject, address) != 0) { - return true; - } - } - return false; - } - - /** - * Returns the index of the first bit that is set to true that occurs on or after the - * specified starting index. If no such bit exists then {@code -1} is returned. - *
- * To iterate over the true bits in a BitSet, use the following loop: - *
- *
- * for (long i = bs.nextSetBit(0, sizeInWords); i >= 0;
- * i = bs.nextSetBit(i + 1, sizeInWords)) {
- * // operate on index i here
- * }
- *
- *
- *
- * @param fromIndex the index to start checking from (inclusive)
- * @param bitSetSizeInWords the size of the bitset, measured in 8-byte words
- * @return the index of the next set bit, or -1 if there is no such bit
- */
- public static int nextSetBit(
- Object baseObject,
- long baseOffset,
- int fromIndex,
- int bitSetSizeInWords) {
- int wi = fromIndex >> 6;
- if (wi >= bitSetSizeInWords) {
- return -1;
- }
-
- // Try to find the next set bit in the current word
- final int subIndex = fromIndex & 0x3f;
- long word = Platform.getLong(baseObject, baseOffset + wi * WORD_SIZE) >> subIndex;
- if (word != 0) {
- return (wi << 6) + subIndex + Long.numberOfTrailingZeros(word);
- }
-
- // Find the next set bit in the rest of the words
- wi += 1;
- while (wi < bitSetSizeInWords) {
- word = Platform.getLong(baseObject, baseOffset + wi * WORD_SIZE);
- if (word != 0) {
- return (wi << 6) + Long.numberOfTrailingZeros(word);
- }
- wi += 1;
- }
-
- return -1;
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/hash/Murmur3OfX86And32Bit.java b/src/main/java/com/actiontech/dble/memory/unsafe/hash/Murmur3OfX86And32Bit.java
deleted file mode 100644
index 747ec067f..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/hash/Murmur3OfX86And32Bit.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.hash;
-
-
-import com.actiontech.dble.memory.unsafe.Platform;
-
-/**
- * 32-bit Murmur3 hasher. This is based on Guava's Murmur3_32HashFunction.
- */
-public final class Murmur3OfX86And32Bit {
- private static final int C1 = 0xcc9e2d51;
- private static final int C2 = 0x1b873593;
-
- private final int seed;
-
- public Murmur3OfX86And32Bit(int seed) {
- this.seed = seed;
- }
-
- @Override
- public String toString() {
- return "Murmur3_32(seed=" + seed + ")";
- }
-
- public int hashInt(int input) {
- return hashInt(input, seed);
- }
-
- public static int hashInt(int input, int seed) {
- int k1 = mixK1(input);
- int h1 = mixH1(seed, k1);
-
- return fMix(h1, 4);
- }
-
- public int hashUnsafeWords(Object base, long offset, int lengthInBytes) {
- return hashUnsafeWords(base, offset, lengthInBytes, seed);
- }
-
- public static int hashUnsafeWords(Object base, long offset, int lengthInBytes, int seed) {
- // This is based on Guava's `Murmur32_Hasher.processRemaining(ByteBuffer)` method.
- assert (lengthInBytes % 8 == 0) : "lengthInBytes must be a multiple of 8 (word-aligned)";
- int h1 = hashBytesByInt(base, offset, lengthInBytes, seed);
- return fMix(h1, lengthInBytes);
- }
-
- public static int hashUnsafeBytes(Object base, long offset, int lengthInBytes, int seed) {
- assert (lengthInBytes >= 0) : "lengthInBytes cannot be negative";
- int lengthAligned = lengthInBytes - lengthInBytes % 4;
- int h1 = hashBytesByInt(base, offset, lengthAligned, seed);
- for (int i = lengthAligned; i < lengthInBytes; i++) {
- int halfWord = Platform.getByte(base, offset + i);
- int k1 = mixK1(halfWord);
- h1 = mixH1(h1, k1);
- }
- return fMix(h1, lengthInBytes);
- }
-
- private static int hashBytesByInt(Object base, long offset, int lengthInBytes, int seed) {
- assert (lengthInBytes % 4 == 0);
- int h1 = seed;
- for (int i = 0; i < lengthInBytes; i += 4) {
- int halfWord = Platform.getInt(base, offset + i);
- int k1 = mixK1(halfWord);
- h1 = mixH1(h1, k1);
- }
- return h1;
- }
-
- public int hashLong(long input) {
- return hashLong(input, seed);
- }
-
- public static int hashLong(long input, int seed) {
- int low = (int) input;
- int high = (int) (input >>> 32);
-
- int k1 = mixK1(low);
- int h1 = mixH1(seed, k1);
-
- k1 = mixK1(high);
- h1 = mixH1(h1, k1);
-
- return fMix(h1, 8);
- }
-
- private static int mixK1(int k1) {
- k1 *= C1;
- k1 = Integer.rotateLeft(k1, 15);
- k1 *= C2;
- return k1;
- }
-
- private static int mixH1(int h1, int k1) {
- h1 ^= k1;
- h1 = Integer.rotateLeft(h1, 13);
- h1 = h1 * 5 + 0xe6546b64;
- return h1;
- }
-
- // Finalization mix - force all bits of a hash block to avalanche
- private static int fMix(int h1, int length) {
- h1 ^= length;
- h1 ^= h1 >>> 16;
- h1 *= 0x85ebca6b;
- h1 ^= h1 >>> 13;
- h1 *= 0xc2b2ae35;
- h1 ^= h1 >>> 16;
- return h1;
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/map/BytesToBytesMap.java b/src/main/java/com/actiontech/dble/memory/unsafe/map/BytesToBytesMap.java
deleted file mode 100644
index 94bfe3b63..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/map/BytesToBytesMap.java
+++ /dev/null
@@ -1,978 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.map;
-
-import com.actiontech.dble.memory.unsafe.Platform;
-import com.actiontech.dble.memory.unsafe.array.ByteArrayMethods;
-import com.actiontech.dble.memory.unsafe.array.LongArray;
-import com.actiontech.dble.memory.unsafe.hash.Murmur3OfX86And32Bit;
-import com.actiontech.dble.memory.unsafe.memory.MemoryBlock;
-import com.actiontech.dble.memory.unsafe.memory.mm.DataNodeMemoryManager;
-import com.actiontech.dble.memory.unsafe.memory.mm.MemoryConsumer;
-import com.actiontech.dble.memory.unsafe.storage.DataNodeDiskManager;
-import com.actiontech.dble.memory.unsafe.storage.SerializerManager;
-import com.actiontech.dble.memory.unsafe.utils.sort.UnsafeExternalSorter;
-import com.actiontech.dble.memory.unsafe.utils.sort.UnsafeSorterSpillReader;
-import com.actiontech.dble.memory.unsafe.utils.sort.UnsafeSorterSpillWriter;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.Closeables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.LinkedList;
-
-/**
- * An append-only hash map where keys and values are contiguous regions of bytes.
- * - * This is backed by a power-of-2-sized hash table, using quadratic probing with triangular numbers, - * which is guaranteed to exhaust the space. - *
- * The map can support up to 2^29 keys. If the key cardinality is higher than this, you should - * probably be using sorting instead of hashing for better cache locality. - *
- * The key and values under the hood are stored together, in the following format: - * Bytes 0 to 4: len(k) (key length in bytes) + len(v) (value length in bytes) + 4 - * Bytes 4 to 8: len(k) - * Bytes 8 to 8 + len(k): key data - * Bytes 8 + len(k) to 8 + len(k) + len(v): value data - * Bytes 8 + len(k) + len(v) to 8 + len(k) + len(v) + 8: pointer to next pair - *
- * This means that the first four bytes store the entire record (key + value) length. This format
- * is compatible with {@link UnsafeExternalSorter},
- * so we can pass records from this map directly into the sorter to sort records in place.
- */
-public final class BytesToBytesMap extends MemoryConsumer {
-
- private final Logger logger = LoggerFactory.getLogger(BytesToBytesMap.class);
-
- private static final HashMapGrowthStrategy GROWTH_STRATEGY = HashMapGrowthStrategy.DOUBLING;
-
- private final DataNodeMemoryManager dataNodeMemoryManager;
-
- /**
- * A linked list for tracking all allocated data pages so that we can free all of our memory.
- */
- private final LinkedList
- * Position {@code 2 * i} in the array is used to track a pointer to the key at index {@code i},
- * while position {@code 2 * i + 1} in the array holds key's full 32-bit hashcode.
- */
- @Nullable
- private LongArray longArray;
- // TODO: we're wasting 32 bits of space here; we can probably store fewer bits of the hashcode
- // and exploit word-alignment to use fewer bits to hold the address. This might let us store
- // only one long per map entry, increasing the chance that this array will fit in cache at the
- // expense of maybe performing more lookups if we have hash collisions. Say that we stored only
- // 27 bits of the hashcode and 37 bits of the address. 37 bits is enough to address 1 terabyte
- // of RAM given word-alignment. If we use 13 bits of this for our page table, that gives us a
- // maximum page size of 2^24 * 8 = ~134 megabytes per page. This change will require us to store
- // full base addresses in the page table for off-heap mode so that we can reconstruct the full
- // absolute memory addresses.
-
- /**
- * Whether or not the longArray can grow. We will not insert more elements if it's false.
- */
- private boolean canGrowArray = true;
-
- private final double loadFactor;
-
- /**
- * The size of the data pages that hold key and value data. Map entries cannot span multiple
- * pages, so this limits the maximum entry size.
- */
- private final long pageSizeBytes;
-
- /**
- * Number of keys defined in the map.
- */
- private int numKeys;
-
- /**
- * Number of values defined in the map. A key could have multiple values.
- */
- private int numValues;
-
- /**
- * The map will be expanded once the number of keys exceeds this threshold.
- */
- private int growthThreshold;
-
- /**
- * Mask for truncating hashcodes so that they do not exceed the long array's size.
- * This is a strength reduction optimization; we're essentially performing a modulus operation,
- * but doing so with a bitmask because this is a power-of-2-sized hash map.
- */
- private int mask;
-
- /**
- * Return value of {@link BytesToBytesMap#lookup(Object, long, int)}.
- */
- private final Location loc;
-
- private final boolean enablePerfMetrics;
-
- private long timeSpentResizingNs = 0;
-
- private long numProbes = 0;
-
- private long numKeyLookups = 0;
-
- private long numHashCollisions = 0;
-
- private long peakMemoryUsedBytes = 0L;
-
- private final DataNodeDiskManager blockManager;
- private final SerializerManager serializerManager;
- private volatile MapIterator destructiveIterator = null;
- private LinkedList
- * For efficiency, all calls to `next()` will return the same {@link Location} object.
- *
- * If any other lookups or operations are performed on this map while iterating over it, including
- * `lookup()`, the behavior of the returned iterator is undefined.
- */
- public MapIterator iterator() {
- return new MapIterator(numValues, loc, false);
- }
-
- /**
- * Returns a destructive iterator for iterating over the entries of this map. It frees each page
- * as it moves onto next one. Notice: it is illegal to call any method on the map after
- * `destructiveIterator()` has been called.
- *
- * For efficiency, all calls to `next()` will return the same {@link Location} object.
- *
- * If any other lookups or operations are performed on this map while iterating over it, including
- * `lookup()`, the behavior of the returned iterator is undefined.
- */
- public MapIterator destructiveIterator() {
- return new MapIterator(numValues, loc, true);
- }
-
- /**
- * Looks up a key, and return a {@link Location} handle that can be used to map existence
- * and read/write values.
- *
- * This function always return the same {@link Location} instance to avoid object allocation.
- */
- public Location lookup(Object keyBase, long keyOffset, int keyLength) {
- safeLookup(keyBase, keyOffset, keyLength, loc,
- Murmur3OfX86And32Bit.hashUnsafeWords(keyBase, keyOffset, keyLength, 42));
- return loc;
- }
-
- /**
- * Looks up a key, and return a {@link Location} handle that can be used to map existence
- * and read/write values.
- *
- * This function always return the same {@link Location} instance to avoid object allocation.
- */
- public Location lookup(Object keyBase, long keyOffset, int keyLength, int hash) {
- safeLookup(keyBase, keyOffset, keyLength, loc, hash);
- return loc;
- }
-
- /**
- * Looks up a key, and saves the result in provided `loc`.
- *
- * This is a thread-safe version of `lookup`, could be used by multiple threads.
- */
- public void safeLookup(Object keyBase, long keyOffset, int keyLength, Location location, int hash) {
- assert (longArray != null);
-
- if (enablePerfMetrics) {
- numKeyLookups++;
- }
-
- int pos = hash & mask;
- int step = 1;
-
- while (true) {
- if (enablePerfMetrics) {
- numProbes++;
- }
- if (longArray.get(pos * 2) == 0) {
- // This is a new key.
- location.with(pos, hash, false);
-
- return;
-
- } else {
-
- long stored = longArray.get(pos * 2 + 1);
-
- /**
- * hash is equal
- */
- if ((int) (stored) == hash) {
- // Full hash code matches.Let's compare the keys for equality.
- location.with(pos, hash, true);
- /**
- * compare the key
- */
- if (location.getKeyLength() == keyLength) {
- final boolean areEqual = ByteArrayMethods.arrayEquals(
- keyBase,
- keyOffset,
- location.getKeyBase(),
- location.getKeyOffset(),
- keyLength
- );
-
- if (areEqual) {
-
- return;
- } else {
- if (enablePerfMetrics) {
- numHashCollisions++;
- }
- }
- }
- }
- }
- pos = (pos + step) & mask;
- step++;
- }
- }
-
- /**
- * Acquire a new page from the memory manager.
- *
- * @return whether there is enough space to allocate the new page.
- */
- private boolean acquireNewPage(long required) {
- try {
- currentPage = allocatePage(required);
- } catch (OutOfMemoryError e) {
- return false;
- }
- dataPages.add(currentPage);
- Platform.putInt(currentPage.getBaseObject(), currentPage.getBaseOffset(), 0);
- pageCursor = 4;
- return true;
- }
-
- @Override
- public long spill(long size, MemoryConsumer trigger) throws IOException {
- if (trigger != this && destructiveIterator != null) {
- return destructiveIterator.spill(size);
- }
- return 0L;
- }
-
- /**
- * Allocate new data structures for this map. When calling this outside of the constructor,
- * make sure to keep references to the old data structures so that you can free them.
- *
- * @param capacity the new map capacity
- */
- private void allocate(int capacity) {
- assert (capacity >= 0);
- capacity = Math.max((int) Math.min(MAX_CAPACITY, ByteArrayMethods.nextPowerOf2(capacity)), 64);
- assert (capacity <= MAX_CAPACITY);
- longArray = allocateLongArray(capacity * 2);
- longArray.zeroOut();
-
- this.growthThreshold = (int) (capacity * loadFactor);
- this.mask = capacity - 1;
- }
-
- /**
- * Free all allocated memory associated with this map, including the storage for keys and values
- * as well as the hash map array itself.
- *
- * This method is idempotent and can be called multiple times.
- */
- public void free() {
- updatePeakMemoryUsed();
- if (longArray != null) {
- freeLongArray(longArray);
- longArray = null;
- }
- Iterator
- * It is only valid to call this method immediately after calling `lookup()` using the same key.
- *
- * The key and value must be word-aligned (that is, their sizes must multiples of 8).
- *
- * After calling this method, calls to `get[Key|Value]Address()` and `get[Key|Value]Length`
- * will return information on the data stored by this `append` call.
- *
- * As an example usage, here's the proper way to store a new key:
- *
- * Unspecified behavior if the key is not defined.
- *
- * For efficiency, each call returns the same object.
- */
- public KVIterator
- * Most of the complexity in this class deals with encoding of off-heap addresses into 64-bit longs.
- * In off-heap mode, memory can be directly addressed with 64-bit longs. In on-heap mode, memory is
- * addressed by the combination of a base Object reference and a 64-bit offset within that object.
- * This is a problem when we want to store pointers to data structures inside of other structures,
- * such as record pointers inside hashmaps or sorting buffers. Even if we decided to use 128 bits
- * to address memory, we can't just store the address of the base object since it's not guaranteed
- * to remain stable as the heap gets reorganized due to GC.
- *
- * Instead, we use the following approach to encode record pointers in 64-bit longs: for off-heap
- * mode, just store the raw address, and for on-heap mode use the upper 13 bits of the address to
- * store a "page number" and the lower 51 bits to store an offset within this page. These page
- * numbers are used to index into a "page table" array inside of the MemoryManager in order to
- * retrieve the base object.
- *
- * This allows us to address 8192 pages. In on-heap mode, the maximum page size is limited by the
- * maximum size of a long[] array, allowing us to address 8192 * 2^32 * 8 bytes, which is
- * approximately 35 terabytes of memory.
- */
-public class DataNodeMemoryManager {
-
- private final Logger logger = LoggerFactory.getLogger(DataNodeMemoryManager.class);
-
- /**
- * The number of bits used to address the page table.
- */
- private static final int PAGE_NUMBER_BITS = 13;
-
- /**
- * The number of bits used to encode offsets in data pages.
- */
- public static final int OFFSET_BITS = 64 - PAGE_NUMBER_BITS; // 51
-
- /**
- * The number of entries in the page table.
- */
- private static final int PAGE_TABLE_SIZE = 1 << PAGE_NUMBER_BITS;
-
- /**
- * Maximum supported data page size (in bytes). In principle, the maximum addressable page size is
- * (1L << OFFSET_BITS) bytes, which is 2+ petabytes. However, the on-heap allocator's
- * maximum page size is limited by the maximum amount of data that can be stored in a long[]
- * array, which is (2^32 - 1) * 8 bytes (or 16 gigabytes). Therefore, we cap this at 16 gigabytes.
- */
- public static final long MAXIMUM_PAGE_SIZE_BYTES = ((1L << 31) - 1) * 8L;
-
- /**
- * Bit mask for the lower 51 bits of a long.
- */
- private static final long MASK_LONG_LOWER_51_BITS = 0x7FFFFFFFFFFFFL;
-
- /**
- * Bit mask for the upper 13 bits of a long
- */
- private static final long MASK_LONG_UPPER_13_BITS = ~MASK_LONG_LOWER_51_BITS;
-
- /**
- * Similar to an operating system's page table, this array maps page numbers into base object
- * pointers, allowing us to translate between the hashtable's internal 64-bit address
- * representation and the baseObject+offset representation which we use to support both in- and
- * off-heap addresses. When using an off-heap allocator, every entry in this map will be `null`.
- * When using an in-heap allocator, the entries in this map will point to pages' base objects.
- * Entries are added to this map as new data pages are allocated.
- */
- private final MemoryBlock[] pageTable = new MemoryBlock[PAGE_TABLE_SIZE];
-
- /**
- * Bitmap for tracking free pages.
- */
- private final BitSet allocatedPages = new BitSet(PAGE_TABLE_SIZE);
-
- private final MemoryManager memoryManager;
-
- private final long connectionAttemptId;
-
- /**
- * Tracks whether we're in-heap or off-heap. For off-heap, we short-circuit most of these methods
- * without doing any masking or lookups. Since this branching should be well-predicted by the JIT,
- * this extra layer of indirection / abstraction hopefully shouldn't be too expensive.
- */
- public final MemoryMode tungstenMemoryMode;
-
- /**
- * Tracks spillable memory consumers.
- */
- @GuardedBy("this")
- private final HashSet
- * Returns `null` if there was not enough memory to allocate the page. May return a page that
- * contains fewer bytes than requested, so callers should verify the size of returned pages.
- */
- public MemoryBlock allocatePage(long size, MemoryConsumer consumer) {
- if (size > MAXIMUM_PAGE_SIZE_BYTES) {
- throw new IllegalArgumentException(
- "Cannot allocate a page with more than " + MAXIMUM_PAGE_SIZE_BYTES + " bytes");
- }
-
- /**
- * spill to disk ,release the memory
- */
- long acquired = 0;
- try {
- acquired = acquireExecutionMemory(size, tungstenMemoryMode, consumer);
- } catch (InterruptedException e) {
- logger.error(e.getMessage());
- }
-
- if (acquired <= 0) {
- return null;
- }
-
- final int pageNumber;
-
- synchronized (this) {
- pageNumber = allocatedPages.nextClearBit(0);
- if (pageNumber >= PAGE_TABLE_SIZE) {
- releaseExecutionMemory(acquired, tungstenMemoryMode, consumer);
- throw new IllegalStateException(
- "Have already allocated a maximum of " + PAGE_TABLE_SIZE + " pages");
- }
- allocatedPages.set(pageNumber);
- }
-
-
- MemoryBlock page = null;
-
- try {
- page = memoryManager.tungstenMemoryAllocator().allocate(acquired);
- } catch (OutOfMemoryError e) {
- logger.warn("Failed to allocate a page ({} bytes), try again.", acquired);
- // there is no enough memory actually, it means the actual free memory is smaller than
- // MemoryManager thought, we should keep the acquired memory.
- synchronized (this) {
- acquiredButNotUsed += acquired;
- allocatedPages.clear(pageNumber);
- }
- // this could trigger spilling to free some pages.
- return allocatePage(size, consumer);
- }
-
- page.setPageNumber(pageNumber);
- pageTable[pageNumber] = page;
-
- return page;
- }
-
- /**
- * Free a block of memory allocated via {@link DataNodeMemoryManager#allocatePage}.
- */
- public void freePage(MemoryBlock page, MemoryConsumer consumer) {
-
- assert (page.getPageNumber() != -1) :
- "Called freePage() on memory that wasn't allocated with allocatePage()";
- assert (allocatedPages.get(page.getPageNumber()));
- pageTable[page.getPageNumber()] = null;
-
- synchronized (this) {
- allocatedPages.clear(page.getPageNumber());
- }
-
- logger.trace("Freed page number " + page.getPageNumber() + " (" + page.size() + " bytes)");
-
- long pageSize = page.size();
- memoryManager.tungstenMemoryAllocator().free(page);
- releaseExecutionMemory(pageSize, tungstenMemoryMode, consumer);
- }
-
- /**
- * Given a memory page and offset within that page, encode this address into a 64-bit long.
- * This address will remain valid as long as the corresponding page has not been freed.
- *
- * @param page a data page allocated by {@link DataNodeMemoryManager#allocatePage}/
- * @param offsetInPage an offset in this page which incorporates the base offset. In other words,
- * this should be the value that you would pass as the base offset into an
- * UNSAFE call (e.g. page.baseOffset() + something).
- * @return an encoded page address.
- */
- public long encodePageNumberAndOffset(MemoryBlock page, long offsetInPage) {
-
- if (tungstenMemoryMode == MemoryMode.OFF_HEAP) {
- // In off-heap mode, an offset is an absolute address that may require a full 64 bits to
- // encode. Due to our page size limitation, though, we can convert this into an offset that's
- // relative to the page's base offset; this relative offset will fit in 51 bits.
- offsetInPage -= page.getBaseOffset();
- }
-
- return encodePageNumberAndOffset(page.getPageNumber(), offsetInPage);
- }
-
- @VisibleForTesting
- public static long encodePageNumberAndOffset(int pageNumber, long offsetInPage) {
- assert (pageNumber != -1) : "encodePageNumberAndOffset called with invalid page";
- return (((long) pageNumber) << OFFSET_BITS) | (offsetInPage & MASK_LONG_LOWER_51_BITS);
- }
-
- @VisibleForTesting
- public static int decodePageNumber(long pagePlusOffsetAddress) {
- return (int) (pagePlusOffsetAddress >>> OFFSET_BITS);
- }
-
- private static long decodeOffset(long pagePlusOffsetAddress) {
- return (pagePlusOffsetAddress & MASK_LONG_LOWER_51_BITS);
- }
-
- /**
- * Get the page associated with an address encoded by
- * {@link DataNodeMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)}
- */
- public Object getPage(long pagePlusOffsetAddress) {
- if (tungstenMemoryMode == MemoryMode.ON_HEAP) {
- final int pageNumber = decodePageNumber(pagePlusOffsetAddress);
- assert (pageNumber >= 0 && pageNumber < PAGE_TABLE_SIZE);
- final MemoryBlock page = pageTable[pageNumber];
- assert (page != null);
- assert (page.getBaseObject() != null);
- return page.getBaseObject();
- } else {
- return null;
- }
- }
-
- /**
- * Get the offset associated with an address encoded by
- * {@link DataNodeMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)}
- */
- public long getOffsetInPage(long pagePlusOffsetAddress) {
- final long offsetInPage = decodeOffset(pagePlusOffsetAddress);
- if (tungstenMemoryMode == MemoryMode.ON_HEAP) {
- return offsetInPage;
- } else {
- // In off-heap mode, an offset is an absolute address. In encodePageNumberAndOffset, we
- // converted the absolute address into a relative address. Here, we invert that operation:
- final int pageNumber = decodePageNumber(pagePlusOffsetAddress);
- assert (pageNumber >= 0 && pageNumber < PAGE_TABLE_SIZE);
- final MemoryBlock page = pageTable[pageNumber];
- assert (page != null);
- return page.getBaseOffset() + offsetInPage;
- }
- }
-
- /**
- * Clean up all allocated memory and pages. Returns the number of bytes freed. A non-zero return
- * value can be used to detect memory leaks.
- */
- public long cleanUpAllAllocatedMemory() {
- synchronized (this) {
- for (MemoryConsumer c : consumers) {
- if (c != null && c.getUsed() > 0) {
- // In case of failed task, it's normal to see leaked memory
- logger.warn("leak " + JavaUtils.bytesToString(c.getUsed()) + " memory from " + c);
- }
- }
- consumers.clear();
-
- for (MemoryBlock page : pageTable) {
- if (page != null) {
- logger.warn("leak a page: " + page + " in task " + connectionAttemptId);
- memoryManager.tungstenMemoryAllocator().free(page);
- }
- }
- Arrays.fill(pageTable, null);
- }
-
- // release the memory that is not used by any consumer.
- memoryManager.releaseExecutionMemory(acquiredButNotUsed, connectionAttemptId, tungstenMemoryMode);
-
- return memoryManager.releaseAllExecutionMemoryForConnection(connectionAttemptId);
- }
-
- /**
- * Returns the memory consumption, in bytes, for the current task.
- */
- public long getMemoryConsumptionForThisConnection() {
- return memoryManager.getExecutionMemoryUsageForConnection(connectionAttemptId);
- }
-
- /**
- * Returns Tungsten memory mode
- */
- public MemoryMode getTungstenMemoryMode() {
- return tungstenMemoryMode;
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/memory/mm/MemoryConsumer.java b/src/main/java/com/actiontech/dble/memory/unsafe/memory/mm/MemoryConsumer.java
deleted file mode 100644
index d70c7a937..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/memory/mm/MemoryConsumer.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.actiontech.dble.memory.unsafe.memory.mm;
-
-import com.actiontech.dble.memory.unsafe.array.CharArray;
-import com.actiontech.dble.memory.unsafe.array.LongArray;
-import com.actiontech.dble.memory.unsafe.memory.MemoryBlock;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * An memory consumer of DataNodeMemoryManager, which support spilling.
- * Note: this only supports allocation / spilling of Tungsten memory.
- */
-public abstract class MemoryConsumer {
- private final Logger logger = LoggerFactory.getLogger(MemoryConsumer.class);
-
- protected final DataNodeMemoryManager dataNodeMemoryManager;
- private final long pageSize;
- protected long used;
-
- protected MemoryConsumer(DataNodeMemoryManager dataNodeMemoryManager, long pageSize) {
- this.dataNodeMemoryManager = dataNodeMemoryManager;
- this.pageSize = pageSize;
- }
-
- protected MemoryConsumer(DataNodeMemoryManager dataNodeMemoryManager) {
- this(dataNodeMemoryManager, dataNodeMemoryManager.pageSizeBytes());
- }
-
- /**
- * Returns the size of used memory in bytes.
- */
- public long getUsed() {
- return used;
- }
-
- /**
- * Force spill during building.
- *
- * For testing.
- */
- public void spill() throws IOException {
- spill(Long.MAX_VALUE, this);
- }
-
- /**
- * Spill some data to disk to release memory, which will be called by DataNodeMemoryManager
- * when there is not enough memory for the task.
- *
- * This should be implemented by subclass.
- *
- * Note: In order to avoid possible deadlock, should not call acquireMemory() from spill().
- *
- * Note: today, this only frees Tungsten-managed pages.
- *
- * @param size the amount of memory should be released
- * @param trigger the MemoryConsumer that trigger this spilling
- * @return the amount of released memory in bytes
- * @throws IOException
- */
- public abstract long spill(long size, MemoryConsumer trigger) throws IOException;
-
- /**
- * Allocates a LongArray of `size`.
- */
- public LongArray allocateLongArray(long size) {
- long required = size * 8L;
- MemoryBlock page = dataNodeMemoryManager.allocatePage(required, this);
- if (page == null || page.size() < required) {
- long got = 0;
- if (page != null) {
- got = page.size();
- dataNodeMemoryManager.freePage(page, this);
- }
- dataNodeMemoryManager.showMemoryUsage();
- throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got);
- }
- used += required;
- return new LongArray(page);
- }
-
- /**
- * Frees a LongArray.
- */
- public void freeLongArray(LongArray array) {
- freePage(array.memoryBlock());
- }
-
- public CharArray allocateCharArray(long size) {
- long required = size * 2L;
- MemoryBlock page = dataNodeMemoryManager.allocatePage(required, this);
- if (page == null || page.size() < required) {
- long got = 0;
- if (page != null) {
- got = page.size();
- dataNodeMemoryManager.freePage(page, this);
- }
- dataNodeMemoryManager.showMemoryUsage();
- throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got);
- }
- used += required;
- return new CharArray(page);
- }
-
- /**
- * Frees a CharArray.
- */
- public void freeCharArray(CharArray array) {
- freePage(array.memoryBlock());
- }
-
- /**
- * Allocate a memory block with at least `required` bytes.
- *
- * Throws IOException if there is not enough memory.
- *
- * @throws OutOfMemoryError
- */
- protected MemoryBlock allocatePage(long required) {
- MemoryBlock page = dataNodeMemoryManager.allocatePage(Math.max(pageSize, required), this);
- if (page == null || page.size() < required) {
- long got = 0;
- if (page != null) {
- got = page.size();
- dataNodeMemoryManager.freePage(page, this);
- }
- dataNodeMemoryManager.showMemoryUsage();
- throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got);
- }
- used += page.size();
- return page;
- }
-
- /**
- * Free a memory block.
- */
- protected void freePage(MemoryBlock page) {
- used -= page.size();
- dataNodeMemoryManager.freePage(page, this);
- }
-
- /**
- * Allocates a heap memory of `size`.
- */
- public long acquireOnHeapMemory(long size) {
- long granted = 0;
- try {
- granted = dataNodeMemoryManager.acquireExecutionMemory(size, MemoryMode.ON_HEAP, this);
- } catch (InterruptedException e) {
- logger.error(e.getMessage());
- }
- used += granted;
- return granted;
- }
-
- /**
- * Release N bytes of heap memory.
- */
- public void freeOnHeapMemory(long size) {
- dataNodeMemoryManager.releaseExecutionMemory(size, MemoryMode.ON_HEAP, this);
- used -= size;
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/memory/mm/MemoryManager.java b/src/main/java/com/actiontech/dble/memory/unsafe/memory/mm/MemoryManager.java
deleted file mode 100644
index c852bfb64..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/memory/mm/MemoryManager.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Copyright (C) 2016-2017 ActionTech.
- * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
- */
-
-package com.actiontech.dble.memory.unsafe.memory.mm;
-
-
-import com.actiontech.dble.memory.unsafe.Platform;
-import com.actiontech.dble.memory.unsafe.array.ByteArrayMethods;
-import com.actiontech.dble.memory.unsafe.memory.MemoryAllocator;
-import com.actiontech.dble.memory.unsafe.utils.ServerPropertyConf;
-
-import javax.annotation.concurrent.GuardedBy;
-import java.util.concurrent.ConcurrentMap;
-
-public abstract class MemoryManager {
-
- private ServerPropertyConf conf;
-
- @GuardedBy("this")
- protected ResultSetMemoryPool onHeapExecutionMemoryPool =
- new ResultSetMemoryPool(this, MemoryMode.ON_HEAP);
-
- @GuardedBy("this")
- protected ResultSetMemoryPool offHeapExecutionMemoryPool =
- new ResultSetMemoryPool(this, MemoryMode.OFF_HEAP);
-
- protected long maxOffHeapMemory = 0L;
- protected long offHeapExecutionMemory = 0L;
- private int numCores = 0;
-
- public MemoryManager(ServerPropertyConf conf, int numCores, long onHeapExecutionMemory) {
- this.conf = conf;
- this.numCores = numCores;
- maxOffHeapMemory = conf.getSizeAsBytes("server.memory.offHeap.size", "128m");
- offHeapExecutionMemory = maxOffHeapMemory;
- onHeapExecutionMemoryPool.incrementPoolSize(onHeapExecutionMemory);
-
- offHeapExecutionMemoryPool.incrementPoolSize(offHeapExecutionMemory);
- }
-
- protected abstract long acquireExecutionMemory(long numBytes, long taskAttemptId, MemoryMode memoryMode) throws InterruptedException;
-
- /**
- * Release numBytes of execution memory belonging to the given task.
- */
- public void releaseExecutionMemory(long numBytes, long taskAttemptId, MemoryMode memoryMode) {
- synchronized (this) {
- if (memoryMode == MemoryMode.ON_HEAP) {
- onHeapExecutionMemoryPool.releaseMemory(numBytes, taskAttemptId);
-
- } else if (memoryMode == MemoryMode.OFF_HEAP) {
- offHeapExecutionMemoryPool.releaseMemory(numBytes, taskAttemptId);
-
- }
- }
-
- }
-
- /**
- * Release all memory for the given task and mark it as inactive (e.g. when a task ends).
- *
- * @return the number of bytes freed.
- */
- public long releaseAllExecutionMemoryForConnection(long connAttemptId) {
- synchronized (this) {
- return (onHeapExecutionMemoryPool.releaseAllMemoryForeConnection(connAttemptId) +
- offHeapExecutionMemoryPool.releaseAllMemoryForeConnection(connAttemptId));
- }
- }
-
- /**
- * Execution memory currently in use, in bytes.
- */
- public final long executionMemoryUsed() {
- synchronized (this) {
- return (onHeapExecutionMemoryPool.memoryUsed() + offHeapExecutionMemoryPool.memoryUsed());
- }
- }
-
- /**
- * Returns the execution memory consumption, in bytes, for the given task.
- */
- public long getExecutionMemoryUsageForConnection(long connAttemptId) {
- synchronized (this) {
- assert (connAttemptId >= 0);
- return (onHeapExecutionMemoryPool.getMemoryUsageConnection(connAttemptId) +
- offHeapExecutionMemoryPool.getMemoryUsageConnection(connAttemptId));
- }
- }
-
- /**
- * Tracks whether Tungsten memory will be allocated on the JVM heap or off-heap using
- * sun.misc.Unsafe.
- */
- public final MemoryMode tungstenMemoryMode() {
- if (conf.getBoolean("server.memory.offHeap.enabled", false)) {
- assert (conf.getSizeAsBytes("server.memory.offHeap.size", 0) > 0);
- assert (Platform.unaligned());
- return MemoryMode.OFF_HEAP;
- } else {
- return MemoryMode.ON_HEAP;
- }
- }
-
- /**
- * The default page size, in bytes.
- *
- * If user didn't explicitly set "server.buffer.pageSize", we figure out the default value
- * by looking at the number of cores available to the process, and the total amount of memory,
- * and then divide it by a factor of safety.
- */
- public long pageSizeBytes() {
-
- long minPageSize = 1L * 1024 * 1024; // 1MB
- long maxPageSize = 64L * minPageSize; // 64MB
-
- int cores = 0;
-
- if (numCores > 0) {
- cores = numCores;
- } else {
- cores = Runtime.getRuntime().availableProcessors();
- }
-
- // Because of rounding to next power of 2, we may have safetyFactor as 8 in worst case
- int safetyFactor = 16;
- long maxTungstenMemory = 0L;
-
- MemoryMode i = tungstenMemoryMode();
- if (i == MemoryMode.ON_HEAP) {
- synchronized (this) {
- maxTungstenMemory = onHeapExecutionMemoryPool.poolSize();
- }
-
- } else if (i == MemoryMode.OFF_HEAP) {
- synchronized (this) {
- maxTungstenMemory = offHeapExecutionMemoryPool.poolSize();
- }
-
- }
-
- long size = ByteArrayMethods.nextPowerOf2(maxTungstenMemory / cores / safetyFactor);
- long defaultSize = Math.min(maxPageSize, Math.max(minPageSize, size));
- defaultSize = conf.getSizeAsBytes("server.buffer.pageSize", defaultSize);
-
- return defaultSize;
- }
-
- /**
- * Allocates memory for use by Unsafe/Tungsten code.
- */
- public final MemoryAllocator tungstenMemoryAllocator() {
- MemoryMode i = tungstenMemoryMode();
- if (i == MemoryMode.ON_HEAP) {
- return MemoryAllocator.HEAP;
- } else if (i == MemoryMode.OFF_HEAP) {
- return MemoryAllocator.UNSAFE;
- }
- return null;
- }
-
- /**
- * Get Direct Memory Usage.
- */
- public final ConcurrentMap
- * This call may block until there is enough free memory in some situations, to make sure each
- * task has a chance to ramp up to at least 1 / 8N of the total memory pool (where N is the # of
- * active tasks) before it is forced to spill. This can happen if the number of tasks increase
- * but an older task had a lot of memory already.
- *
- * @param numBytes number of bytes to acquire
- * @param connAttemptId the task attempt acquiring memory
- * @return the number of bytes granted to the task.
- */
- public long acquireMemory(long numBytes, long connAttemptId) throws InterruptedException {
-
- synchronized (lock) {
- assert (numBytes > 0);
- // Add this connection to the taskMemory map just so we can keep an accurate count of the number
- // of active tasks, to let other tasks ramp down their memory in calls to `acquireMemory`
- if (!memoryForConnection.containsKey(connAttemptId)) {
- memoryForConnection.put(connAttemptId, 0L);
- // This will later cause waiting tasks to wake up and check numTasks again
- lock.notifyAll();
- }
-
-
- while (true) {
- long numActiveCons = memoryForConnection.size();
- long curMem = memoryForConnection.get(connAttemptId);
-
- long maxPoolSize = poolSize();
- long maxMemoryPerTask = maxPoolSize / numActiveCons;
- long minMemoryPerTask = poolSize() / (8 * numActiveCons);
-
- // How much we can grant this connection; keep its share within 0 <= X <= 1 / numActiveConns
- long maxToGrant = Math.min(numBytes, Math.max(0, maxMemoryPerTask - curMem));
- // Only give it as much memory as is free, which might be none if it reached 1 / numActiveConns
- long toGrant = Math.min(maxToGrant, memoryFree());
-
- // We want to let each connection get at least 1 / (8 * numActiveConns) before blocking;
- // if we can't give it this much now, wait for other tasks to free up memory
- // (this happens if older tasks allocated lots of memory before N grew)
- if (toGrant < numBytes && curMem + toGrant < minMemoryPerTask) {
- LOG.info("Thread " + connAttemptId + " waiting for at least 1/8N of " + poolName() + " pool to be free");
- lock.wait();
- } else {
- long temp = memoryForConnection.get(connAttemptId);
- memoryForConnection.put(connAttemptId, (temp + toGrant));
- return toGrant;
- }
- }
- }
- }
-
- /**
- * Release `numBytes` of memory acquired by the given task.
- */
- public void releaseMemory(long numBytes, long connAttemptId) {
-
- synchronized (lock) {
- long curMem = memoryForConnection.get(connAttemptId);
-
- long memoryToFree = 0L;
-
- if (curMem < numBytes) {
- LOG.warn(
- "Internal error: release called on $numBytes bytes but task only has $curMem bytes " +
- "of memory from the " + poolName() + " pool");
- memoryToFree = curMem;
- } else {
- memoryToFree = numBytes;
- }
-
- if (memoryForConnection.containsKey(connAttemptId)) {
- long temp = memoryForConnection.get(connAttemptId);
- memoryForConnection.put(connAttemptId, (temp - memoryToFree));
- if (memoryForConnection.get(connAttemptId) <= 0) {
- memoryForConnection.remove(connAttemptId);
- }
- }
- // Notify waiters in acquireMemory() that memory has been freed
- lock.notifyAll();
- }
- }
-
- /**
- * Release all memory for the given task and mark it as inactive (e.g. when a task ends).
- *
- * @return the number of bytes freed.
- */
- public long releaseAllMemoryForeConnection(long connAttemptId) {
- synchronized (lock) {
- long numBytesToFree = getMemoryUsageConnection(connAttemptId);
- releaseMemory(numBytesToFree, connAttemptId);
- return numBytesToFree;
- }
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/row/BufferHolder.java b/src/main/java/com/actiontech/dble/memory/unsafe/row/BufferHolder.java
deleted file mode 100644
index 741466dbf..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/row/BufferHolder.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.row;
-
-
-import com.actiontech.dble.memory.unsafe.Platform;
-
-/**
- * A helper class to manage the data buffer for an unsafe row. The data buffer can grow and
- * automatically re-point the unsafe row to it.
- *
- * This class can be used to build a one-pass unsafe row writing program, i.e. data will be written
- * to the data buffer directly and no extra copy is needed. There should be only one instance of
- * this class per writing program, so that the memory segment/data buffer can be reused. Note that
- * for each incoming record, we should call `reset` of BufferHolder instance before write the record
- * and reuse the data buffer.
- *
- * Generally we should call `UnsafeRow.setTotalSize` and pass in `BufferHolder.totalSize` to update
- * the size of the result row, after writing a record to the buffer. However, we can skip this step
- * if the fields of row are all fixed-length, as the size of result row is also fixed.
- */
-public class BufferHolder {
- private byte[] buffer;
- private int cursor = Platform.BYTE_ARRAY_OFFSET;
-
-
- private final UnsafeRow row;
- private final int fixedSize;
-
- public BufferHolder(UnsafeRow row) {
- this(row, 64);
- }
-
- public BufferHolder(UnsafeRow row, int initialSize) {
- this.fixedSize = UnsafeRow.calculateBitSetWidthInBytes(row.numFields()) + 8 * row.numFields();
- this.buffer = new byte[fixedSize + initialSize];
- this.row = row;
- this.row.pointTo(buffer, buffer.length);
- }
-
- /**
- * Grows the buffer by at least neededSize and points the row to the buffer.
- */
- public void grow(int neededSize) {
- final int length = totalSize() + neededSize;
- if (buffer.length < length) {
- // This will not happen frequently, because the buffer is re-used.
- final byte[] tmp = new byte[length * 2];
- Platform.copyMemory(
- buffer,
- Platform.BYTE_ARRAY_OFFSET,
- tmp,
- Platform.BYTE_ARRAY_OFFSET,
- totalSize());
- buffer = tmp;
- row.pointTo(buffer, buffer.length);
- }
- }
-
- public UnsafeRow getRow() {
- return row;
- }
-
-
- public void reset() {
- cursor = Platform.BYTE_ARRAY_OFFSET + fixedSize;
- }
-
- public int totalSize() {
- return cursor - Platform.BYTE_ARRAY_OFFSET;
- }
-
- public byte[] getBuffer() {
- return buffer;
- }
-
- public void setBuffer(byte[] buffer) {
- this.buffer = buffer;
- }
-
- public int getCursor() {
- return cursor;
- }
-
- public void setCursor(int cursor) {
- this.cursor = cursor;
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/row/StructType.java b/src/main/java/com/actiontech/dble/memory/unsafe/row/StructType.java
deleted file mode 100644
index ad2372dbe..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/row/StructType.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (C) 2016-2017 ActionTech.
- * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
- */
-
-package com.actiontech.dble.memory.unsafe.row;
-
-import com.actiontech.dble.sqlengine.mpp.ColMeta;
-import com.actiontech.dble.sqlengine.mpp.OrderCol;
-
-import javax.annotation.Nonnull;
-import java.util.Map;
-
-/**
- * Created by zagnix on 2016/6/6.
- */
-public class StructType {
-
- private final Map
- * Each tuple has three parts: [null bit set] [values] [variable length portion]
- *
- * The bit set is used for null tracking and is aligned to 8-byte word boundaries. It stores
- * one bit per field.
- *
- * In the `values` region, we store one 8-byte word per field. For fields that hold fixed-length
- * primitive types, such as long, double, or int, we store the value directly in the word. For
- * fields with non-primitive or variable-length values, we store a relative offset (w.r.t. the
- * base address of the row) that points to the beginning of the variable-length field, and length
- * (they are combined into a long).
- *
- * Instances of `UnsafeRow` act as pointers to row data stored in this format.
- */
-public final class UnsafeRow extends MySQLPacket {
-
- //////////////////////////////////////////////////////////////////////////////
- // Static methods
- //////////////////////////////////////////////////////////////////////////////
-
- public static int calculateBitSetWidthInBytes(int numFields) {
- return ((numFields + 63) / 64) * 8;
- }
-
- public static int calculateFixedPortionByteSize(int numFields) {
- return 8 * numFields + calculateBitSetWidthInBytes(numFields);
- }
-
- //////////////////////////////////////////////////////////////////////////////
- // Private fields and methods
- //////////////////////////////////////////////////////////////////////////////
-
- private Object baseObject;
- private long baseOffset;
-
- /**
- * The number of fields in this row, used for calculating the bitset width (and in assertions)
- */
- private int numFields;
-
- /**
- * The size of this row's backing data, in bytes)
- */
- private int sizeInBytes;
-
- /**
- * The width of the null tracking bit set, in bytes
- */
- private int bitSetWidthInBytes;
-
- private long getFieldOffset(int ordinal) {
- return baseOffset + bitSetWidthInBytes + ordinal * 8L;
- }
-
- private void assertIndexIsValid(int index) {
- assert index >= 0 : "index (" + index + ") should >= 0";
- assert index < numFields : "index (" + index + ") should < " + numFields;
- }
-
- //////////////////////////////////////////////////////////////////////////////
- // Public methods
- //////////////////////////////////////////////////////////////////////////////
-
- /**
- * Construct a new UnsafeRow. The resulting row won't be usable until `pointTo()` has been called,
- * since the value returned by this constructor is equivalent to a null pointer.
- *
- * @param numFields the number of fields in this row
- */
- public UnsafeRow(int numFields) {
- this.numFields = numFields;
- this.bitSetWidthInBytes = calculateBitSetWidthInBytes(numFields);
- }
-
- // for serializer
- public UnsafeRow() {
- }
-
- public Object getBaseObject() {
- return baseObject;
- }
-
- public long getBaseOffset() {
- return baseOffset;
- }
-
- public int getSizeInBytes() {
- return sizeInBytes;
- }
-
- public int numFields() {
- return numFields;
- }
-
- /**
- * Update this UnsafeRow to point to different backing data.
- *
- * @param object the base object
- * @param offset the offset within the base object
- * @param size the size of this row's backing data, in bytes
- */
- public void pointTo(Object object, long offset, int size) {
- assert numFields >= 0 : "numFields (" + numFields + ") should >= 0";
- this.baseObject = object;
- this.baseOffset = offset;
- this.sizeInBytes = size;
- }
-
- /**
- * Update this UnsafeRow to point to the underlying byte array.
- *
- * @param buf byte array to point to
- * @param size the number of bytes valid in the byte array
- */
- public void pointTo(byte[] buf, int size) {
- pointTo(buf, Platform.BYTE_ARRAY_OFFSET, size);
- }
-
- public void setTotalSize(int size) {
- this.sizeInBytes = size;
- }
-
- public void setNotNullAt(int i) {
- assertIndexIsValid(i);
- BitSetMethods.unset(baseObject, baseOffset, i);
- }
-
-
- public void setNullAt(int i) {
- assertIndexIsValid(i);
- BitSetMethods.set(baseObject, baseOffset, i);
- // To preserve row equality, zero out the value when setting the column to null.
- // Since this row does does not currently support updates to variable-length values, we don't
- // have to worry about zeroing out that data.
- Platform.putLong(baseObject, getFieldOffset(i), 0);
- }
-
- public void update(int ordinal, Object value) {
- throw new UnsupportedOperationException();
- }
-
- public void setInt(int ordinal, int value) {
- assertIndexIsValid(ordinal);
- setNotNullAt(ordinal);
- Platform.putInt(baseObject, getFieldOffset(ordinal), value);
- }
-
- public void setLong(int ordinal, long value) {
- assertIndexIsValid(ordinal);
- setNotNullAt(ordinal);
- Platform.putLong(baseObject, getFieldOffset(ordinal), value);
- }
-
- public void setDouble(int ordinal, double value) {
- assertIndexIsValid(ordinal);
- setNotNullAt(ordinal);
- if (Double.isNaN(value)) {
- value = Double.NaN;
- }
- Platform.putDouble(baseObject, getFieldOffset(ordinal), value);
- }
-
- public void setBoolean(int ordinal, boolean value) {
- assertIndexIsValid(ordinal);
- setNotNullAt(ordinal);
- Platform.putBoolean(baseObject, getFieldOffset(ordinal), value);
- }
-
- public void setShort(int ordinal, short value) {
- assertIndexIsValid(ordinal);
- setNotNullAt(ordinal);
- Platform.putShort(baseObject, getFieldOffset(ordinal), value);
- }
-
- public void setByte(int ordinal, byte value) {
- assertIndexIsValid(ordinal);
- setNotNullAt(ordinal);
- Platform.putByte(baseObject, getFieldOffset(ordinal), value);
- }
-
- public void setFloat(int ordinal, float value) {
- assertIndexIsValid(ordinal);
- setNotNullAt(ordinal);
- if (Float.isNaN(value)) {
- value = Float.NaN;
- }
- Platform.putFloat(baseObject, getFieldOffset(ordinal), value);
- }
-
-
- public boolean isNullAt(int ordinal) {
- assertIndexIsValid(ordinal);
- return BitSetMethods.isSet(baseObject, baseOffset, ordinal);
- }
-
-
- public boolean getBoolean(int ordinal) {
- assertIndexIsValid(ordinal);
- return Platform.getBoolean(baseObject, getFieldOffset(ordinal));
- }
-
-
- public byte getByte(int ordinal) {
- assertIndexIsValid(ordinal);
- return Platform.getByte(baseObject, getFieldOffset(ordinal));
- }
-
-
- public short getShort(int ordinal) {
- assertIndexIsValid(ordinal);
- return Platform.getShort(baseObject, getFieldOffset(ordinal));
- }
-
-
- public int getInt(int ordinal) {
- assertIndexIsValid(ordinal);
- return Platform.getInt(baseObject, getFieldOffset(ordinal));
- }
-
-
- public long getLong(int ordinal) {
- assertIndexIsValid(ordinal);
- return Platform.getLong(baseObject, getFieldOffset(ordinal));
- }
-
-
- public float getFloat(int ordinal) {
- assertIndexIsValid(ordinal);
- return Platform.getFloat(baseObject, getFieldOffset(ordinal));
- }
-
-
- public double getDouble(int ordinal) {
- assertIndexIsValid(ordinal);
- return Platform.getDouble(baseObject, getFieldOffset(ordinal));
- }
-
-
- public UTF8String getUTF8String(int ordinal) {
- if (isNullAt(ordinal)) return null;
- final long offsetAndSize = getLong(ordinal);
- final int offset = (int) (offsetAndSize >> 32);
- final int size = (int) offsetAndSize;
- return UTF8String.fromAddress(baseObject, baseOffset + offset, size);
- }
-
- public byte[] getBinary(int ordinal) {
- if (isNullAt(ordinal)) {
- return null;
- } else {
- final long offsetAndSize = getLong(ordinal);
- final int offset = (int) (offsetAndSize >> 32);
- final int size = (int) offsetAndSize;
- final byte[] bytes = new byte[size];
- Platform.copyMemory(
- baseObject,
- baseOffset + offset,
- bytes,
- Platform.BYTE_ARRAY_OFFSET,
- size
- );
- return bytes;
- }
- }
-
-
- /**
- * Copies this row, returning a self-contained UnsafeRow that stores its data in an internal
- * byte array rather than referencing data stored in a data page.
- */
- public UnsafeRow copy() {
- UnsafeRow rowCopy = new UnsafeRow(numFields);
- final byte[] rowDataCopy = new byte[sizeInBytes];
- Platform.copyMemory(
- baseObject,
- baseOffset,
- rowDataCopy,
- Platform.BYTE_ARRAY_OFFSET,
- sizeInBytes
- );
- rowCopy.pointTo(rowDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes);
- return rowCopy;
- }
-
- /**
- * Creates an empty UnsafeRow from a byte array with specified numBytes and numFields.
- * The returned row is invalid until we call copyFrom on it.
- */
- public static UnsafeRow createFromByteArray(int numBytes, int numFields) {
- final UnsafeRow row = new UnsafeRow(numFields);
- row.pointTo(new byte[numBytes], numBytes);
- return row;
- }
-
- /**
- * Copies the input UnsafeRow to this UnsafeRow, and resize the underlying byte[] when the
- * input row is larger than this row.
- */
- public void copyFrom(UnsafeRow row) {
- // copyFrom is only available for UnsafeRow created from byte array.
- assert (baseObject instanceof byte[]) && baseOffset == Platform.BYTE_ARRAY_OFFSET;
- if (row.sizeInBytes > this.sizeInBytes) {
- // resize the underlying byte[] if it's not large enough.
- this.baseObject = new byte[row.sizeInBytes];
- }
- Platform.copyMemory(
- row.baseObject, row.baseOffset, this.baseObject, this.baseOffset, row.sizeInBytes);
- // update the sizeInBytes.
- this.sizeInBytes = row.sizeInBytes;
- }
-
- /**
- * Write this UnsafeRow's underlying bytes to the given OutputStream.
- *
- * @param out the stream to write to.
- * @param writeBuffer a byte array for buffering chunks of off-heap data while writing to the
- * output stream. If this row is backed by an on-heap byte array, then this
- * buffer will not be used and may be null.
- */
- public void writeToStream(OutputStream out, byte[] writeBuffer) throws IOException {
- if (baseObject instanceof byte[]) {
- int offsetInByteArray = (int) (Platform.BYTE_ARRAY_OFFSET - baseOffset);
- out.write((byte[]) baseObject, offsetInByteArray, sizeInBytes);
- } else {
- int dataRemaining = sizeInBytes;
- long rowReadPosition = baseOffset;
- while (dataRemaining > 0) {
- int toTransfer = Math.min(writeBuffer.length, dataRemaining);
- Platform.copyMemory(
- baseObject, rowReadPosition, writeBuffer, Platform.BYTE_ARRAY_OFFSET, toTransfer);
- out.write(writeBuffer, 0, toTransfer);
- rowReadPosition += toTransfer;
- dataRemaining -= toTransfer;
- }
- }
- }
-
- @Override
- public int hashCode() {
- return Murmur3OfX86And32Bit.hashUnsafeWords(baseObject, baseOffset, sizeInBytes, 42);
- }
-
- @Override
- public boolean equals(Object other) {
- if (other instanceof UnsafeRow) {
- UnsafeRow o = (UnsafeRow) other;
- return (sizeInBytes == o.sizeInBytes) &&
- ByteArrayMethods.arrayEquals(baseObject, baseOffset, o.baseObject, o.baseOffset,
- sizeInBytes);
- }
- return false;
- }
-
- /**
- * Returns the underlying bytes for this UnsafeRow.
- */
- public byte[] getBytes() {
- if (baseObject instanceof byte[] && baseOffset == Platform.BYTE_ARRAY_OFFSET &&
- (((byte[]) baseObject).length == sizeInBytes)) {
- return (byte[]) baseObject;
- } else {
- byte[] bytes = new byte[sizeInBytes];
- Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes);
- return bytes;
- }
- }
-
- public static final byte NULL_MARK = (byte) 251;
- public static final byte EMPTY_MARK = (byte) 0;
-
- @Override
- public ByteBuffer write(ByteBuffer bb, FrontendConnection c,
- boolean writeSocketIfFull) {
- bb = c.checkWriteBuffer(bb, MySQLPacket.PACKET_HEADER_SIZE, writeSocketIfFull);
- BufferUtil.writeUB3(bb, calcPacketSize());
- bb.put(packetId);
- for (int i = 0; i < numFields; i++) {
- if (!isNullAt(i)) {
- byte[] fv = this.getBinary(i);
- if (fv.length == 0) {
- bb = c.checkWriteBuffer(bb, 1, writeSocketIfFull);
- bb.put(UnsafeRow.EMPTY_MARK);
- } else {
- bb = c.checkWriteBuffer(bb, BufferUtil.getLength(fv),
- writeSocketIfFull);
- BufferUtil.writeLength(bb, fv.length);
- /**
- * write data to Writer Buffer
- */
- bb = c.writeToBuffer(fv, bb);
- }
- } else {
- //Col null value
- bb = c.checkWriteBuffer(bb, 1, writeSocketIfFull);
- bb.put(UnsafeRow.NULL_MARK);
- }
- }
- return bb;
- }
-
- @Override
- public int calcPacketSize() {
- int size = 0;
- for (int i = 0; i < numFields; i++) {
- byte[] v = this.getBinary(i);
- size += (v == null || v.length == 0) ? 1 : BufferUtil.getLength(v);
- }
- return size;
- }
-
- public BigDecimal getDecimal(int ordinal, int scale) {
- if (isNullAt(ordinal)) {
- return null;
- }
- byte[] bytes = getBinary(ordinal);
- BigInteger bigInteger = new BigInteger(bytes);
- BigDecimal javaDecimal = new BigDecimal(bigInteger, scale);
- return javaDecimal;
- }
-
- /**
- * update exist decimal column value to new decimal value
- *
- * NOTE: decimal max precision is limit to 38
- *
- * @param ordinal
- * @param value
- * @param precision
- */
- public void updateDecimal(int ordinal, BigDecimal value) {
- assertIndexIsValid(ordinal);
- // fixed length
- long cursor = getLong(ordinal) >>> 32;
- assert cursor > 0 : "invalid cursor " + cursor;
- // zero-out the bytes
- Platform.putLong(baseObject, baseOffset + cursor, 0L);
- Platform.putLong(baseObject, baseOffset + cursor + 8, 0L);
-
- if (value == null) {
- setNullAt(ordinal);
- // keep the offset for future update
- Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32);
- } else {
-
- final BigInteger integer = value.unscaledValue();
- byte[] bytes = integer.toByteArray();
- assert (bytes.length <= 16);
-
- // Write the bytes to the variable length portion.
- Platform.copyMemory(bytes, Platform.BYTE_ARRAY_OFFSET, baseObject, baseOffset + cursor, bytes.length);
- setLong(ordinal, (cursor << 32) | ((long) bytes.length));
- }
-
- }
-
- /**
- * public Decimal getDecimal(int ordinal, int precision, int scale) {
- * if (isNullAt(ordinal)) {
- * return null;
- * }
- * if (precision <= Decimal.MAX_LONG_DIGITS()) {
- * return Decimal.createUnsafe(getLong(ordinal), precision, scale);
- * } else {
- * byte[] bytes = getBinary(ordinal);
- * BigInteger bigInteger = new BigInteger(bytes);
- * BigDecimal javaDecimal = new BigDecimal(bigInteger, scale);
- * return Decimal.apply(javaDecimal, precision, scale);
- * }
- * }
- *
- * public void setDecimal(int ordinal, Decimal value, int precision) {
- * assertIndexIsValid(ordinal);
- * if (precision <= Decimal.MAX_LONG_DIGITS()) {
- * // compact format
- * if (value == null) {
- * setNullAt(ordinal);
- * } else {
- * setLong(ordinal, value.toUnscaledLong());
- * }
- * } else {
- * // fixed length
- * long cursor = getLong(ordinal) >>> 32;
- * assert cursor > 0 : "invalid cursor " + cursor;
- * // zero-out the bytes
- * Platform.putLong(baseObject, baseOffset + cursor, 0L);
- * Platform.putLong(baseObject, baseOffset + cursor + 8, 0L);
- *
- * if (value == null) {
- * setNullAt(ordinal);
- * // keep the offset for future update
- * Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32);
- * } else {
- *
- * final BigInteger integer = value.toJavaBigDecimal().unscaledValue();
- * byte[] bytes = integer.toByteArray();
- * assert(bytes.length <= 16);
- *
- * // Write the bytes to the variable length portion.
- * Platform.copyMemory(
- * bytes, Platform.BYTE_ARRAY_OFFSET, baseObject, baseOffset + cursor, bytes.length);
- * setLong(ordinal, (cursor << 32) | ((long) bytes.length));
- * }
- * }
- * }
- */
- @Override
- protected String getPacketInfo() {
- return "MySQL RowData Packet";
- }
-
- // This is for debugging
- @Override
- public String toString() {
- StringBuilder build = new StringBuilder("[");
- for (int i = 0; i < sizeInBytes; i += 8) {
- if (i != 0) build.append(',');
- build.append(Long.toHexString(Platform.getLong(baseObject, baseOffset + i)));
- }
- build.append(']');
- return build.toString();
- }
-
- public boolean anyNull() {
- return BitSetMethods.anySet(baseObject, baseOffset, bitSetWidthInBytes / 8);
- }
-
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/row/UnsafeRowWriter.java b/src/main/java/com/actiontech/dble/memory/unsafe/row/UnsafeRowWriter.java
deleted file mode 100644
index 981543bf3..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/row/UnsafeRowWriter.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.row;
-
-
-import com.actiontech.dble.memory.unsafe.Platform;
-import com.actiontech.dble.memory.unsafe.array.ByteArrayMethods;
-import com.actiontech.dble.memory.unsafe.bitset.BitSetMethods;
-
-import java.math.BigDecimal;
-
-/**
- * A helper class to write data into global row buffer using `UnsafeRow` format.
- *
- * It will remember the offset of row buffer which it starts to write, and move the cursor of row
- * buffer while writing. If new data(can be the input record if this is the outermost writer, or
- * nested struct if this is an inner writer) comes, the starting cursor of row buffer may be
- * changed, so we need to call `UnsafeRowWriter.reset` before writing, to update the
- * `startingOffset` and clear out null bits.
- *
- * Note that if this is the outermost writer, which means we will always write from the very
- * beginning of the global row buffer, we don't need to update `startingOffset` and can just call
- * `zeroOutNullBytes` before writing new data.
- */
-public class UnsafeRowWriter {
-
- private final BufferHolder holder;
- // The offset of the global buffer where we start to write this row.
- private int startingOffset;
- private final int nullBitsSize;
- private final int fixedSize;
-
- public UnsafeRowWriter(BufferHolder holder, int numFields) {
- this.holder = holder;
- this.nullBitsSize = UnsafeRow.calculateBitSetWidthInBytes(numFields);
- this.fixedSize = nullBitsSize + 8 * numFields;
- this.startingOffset = holder.getCursor();
- }
-
- /**
- * Resets the `startingOffset` according to the current cursor of row buffer, and clear out null
- * bits. This should be called before we write a new nested struct to the row buffer.
- */
- public void reset() {
- this.startingOffset = holder.getCursor();
-
- // grow the global buffer to make sure it has enough space to write fixed-length data.
- holder.grow(fixedSize);
- holder.setCursor(holder.getCursor() + fixedSize);
-
- zeroOutNullBytes();
- }
-
- /**
- * Clears out null bits. This should be called before we write a new row to row buffer.
- */
- public void zeroOutNullBytes() {
- for (int i = 0; i < nullBitsSize; i += 8) {
- Platform.putLong(holder.getBuffer(), startingOffset + i, 0L);
- }
- }
-
- private void zeroOutPaddingBytes(int numBytes) {
- if ((numBytes & 0x07) > 0) {
- Platform.putLong(holder.getBuffer(), holder.getCursor() + ((numBytes >> 3) << 3), 0L);
- }
- }
-
- public BufferHolder holder() {
- return holder;
- }
-
- public boolean isNullAt(int ordinal) {
- return BitSetMethods.isSet(holder.getBuffer(), startingOffset, ordinal);
- }
-
- public void setNullAt(int ordinal) {
- BitSetMethods.set(holder.getBuffer(), startingOffset, ordinal);
- Platform.putLong(holder.getBuffer(), getFieldOffset(ordinal), 0L);
- }
-
- public long getFieldOffset(int ordinal) {
- return startingOffset + nullBitsSize + 8 * ordinal;
- }
-
- public void setOffsetAndSize(int ordinal, long size) {
- setOffsetAndSize(ordinal, holder.getCursor(), size);
- }
-
- public void setOffsetAndSize(int ordinal, long currentCursor, long size) {
- final long relativeOffset = currentCursor - startingOffset;
- final long fieldOffset = getFieldOffset(ordinal);
- final long offsetAndSize = (relativeOffset << 32) | size;
-
- Platform.putLong(holder.getBuffer(), fieldOffset, offsetAndSize);
- }
-
- // Do word alignment for this row and grow the row buffer if needed.
- // todo: remove this after we make unsafe array data word align.
- public void alignToWords(int numBytes) {
- final int remainder = numBytes & 0x07;
-
- if (remainder > 0) {
- final int paddingBytes = 8 - remainder;
- holder.grow(paddingBytes);
-
- for (int i = 0; i < paddingBytes; i++) {
- Platform.putByte(holder.getBuffer(), holder.getCursor(), (byte) 0);
- holder.setCursor(holder.getCursor() + 1);
- }
- }
- }
-
- public void write(int ordinal, boolean value) {
- final long offset = getFieldOffset(ordinal);
- Platform.putLong(holder.getBuffer(), offset, 0L);
- Platform.putBoolean(holder.getBuffer(), offset, value);
- }
-
- public void write(int ordinal, byte value) {
- final long offset = getFieldOffset(ordinal);
- Platform.putLong(holder.getBuffer(), offset, 0L);
- Platform.putByte(holder.getBuffer(), offset, value);
- }
-
- public void write(int ordinal, short value) {
- final long offset = getFieldOffset(ordinal);
- Platform.putLong(holder.getBuffer(), offset, 0L);
- Platform.putShort(holder.getBuffer(), offset, value);
- }
-
- public void write(int ordinal, int value) {
- final long offset = getFieldOffset(ordinal);
- Platform.putLong(holder.getBuffer(), offset, 0L);
- Platform.putInt(holder.getBuffer(), offset, value);
- }
-
- public void write(int ordinal, long value) {
- Platform.putLong(holder.getBuffer(), getFieldOffset(ordinal), value);
- }
-
- public void write(int ordinal, float value) {
- if (Float.isNaN(value)) {
- value = Float.NaN;
- }
- final long offset = getFieldOffset(ordinal);
- Platform.putLong(holder.getBuffer(), offset, 0L);
- Platform.putFloat(holder.getBuffer(), offset, value);
- }
-
- public void write(int ordinal, double value) {
- if (Double.isNaN(value)) {
- value = Double.NaN;
- }
- Platform.putDouble(holder.getBuffer(), getFieldOffset(ordinal), value);
- }
-
- public void write(int ordinal, byte[] input) {
- if (input == null) {
- return;
- }
- write(ordinal, input, 0, input.length);
- }
-
- public void write(int ordinal, byte[] input, int offset, int numBytes) {
- final int roundedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes);
-
- // grow the global buffer before writing data.
- holder.grow(roundedSize);
-
- zeroOutPaddingBytes(numBytes);
-
- // Write the bytes to the variable length portion.
- Platform.copyMemory(input, Platform.BYTE_ARRAY_OFFSET + offset,
- holder.getBuffer(), holder.getCursor(), numBytes);
-
- setOffsetAndSize(ordinal, numBytes);
-
- // move the cursor forward.
- holder.setCursor(holder.getCursor() + roundedSize);
- }
-
- /**
- * different from Spark, we use java BigDecimal here,
- * and we limit the max precision to be 38 because the bytes length limit to be 16
- *
- * @param ordinal
- * @param input
- */
- public void write(int ordinal, BigDecimal input) {
-
- // grow the global buffer before writing data.
- holder.grow(16);
-
- // zero-out the bytes
- Platform.putLong(holder.getBuffer(), holder.getCursor(), 0L);
- Platform.putLong(holder.getBuffer(), holder.getCursor() + 8, 0L);
-
- // Make sure Decimal object has the same scale as DecimalType.
- // Note that we may pass in null Decimal object to set null for it.
- if (input == null) {
- BitSetMethods.set(holder.getBuffer(), startingOffset, ordinal);
- // keep the offset for future update
- setOffsetAndSize(ordinal, 0L);
- } else {
- final byte[] bytes = input.unscaledValue().toByteArray();
- assert bytes.length <= 16;
-
- // Write the bytes to the variable length portion.
- Platform.copyMemory(bytes, Platform.BYTE_ARRAY_OFFSET, holder.getBuffer(), holder.getCursor(), bytes.length);
- setOffsetAndSize(ordinal, bytes.length);
- }
-
- // move the cursor forward.
- holder.setCursor(holder.getCursor() + 16);
- }
-
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/storage/ConnectionId.java b/src/main/java/com/actiontech/dble/memory/unsafe/storage/ConnectionId.java
deleted file mode 100644
index 78386b9aa..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/storage/ConnectionId.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (C) 2016-2017 ActionTech.
- * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
- */
-
-package com.actiontech.dble.memory.unsafe.storage;
-
-/**
- * Created by zagnix on 2016/6/6.
- */
-public abstract class ConnectionId {
- protected String name;
-
- public abstract String getBlockName();
-
- @Override
- public boolean equals(Object arg0) {
- return super.equals(arg0);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
-
- @Override
- public String toString() {
- return super.toString();
- }
-
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/storage/DataNodeDiskManager.java b/src/main/java/com/actiontech/dble/memory/unsafe/storage/DataNodeDiskManager.java
deleted file mode 100644
index 7fc6f835b..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/storage/DataNodeDiskManager.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2016-2017 ActionTech.
- * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
- */
-
-package com.actiontech.dble.memory.unsafe.storage;
-
-
-import com.actiontech.dble.memory.unsafe.utils.ServerPropertyConf;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- * Created by zagnix on 2016/6/3.
- */
-public class DataNodeDiskManager {
-
- private ServerPropertyConf conf;
- private boolean deleteFilesOnStop;
-
- public DataNodeDiskManager(ServerPropertyConf conf, boolean deleteFilesOnStop) {
- this.conf = conf;
- this.deleteFilesOnStop = deleteFilesOnStop;
- }
-
- public DataNodeFileManager diskBlockManager() throws IOException {
- return new DataNodeFileManager(conf, deleteFilesOnStop);
- }
-
-
- /**
- * A short circuited method to get a block writer that can write data directly to disk.
- * The Block will be appended to the File specified by filename. Callers should handle error
- * cases.
- */
- public DiskRowWriter getDiskWriter(
- File file,
- SerializerInstance serializerInstance,
- int bufferSize) throws IOException {
- boolean syncWrites = conf.getBoolean("server.merge.sync", false);
- return new DiskRowWriter(file, serializerInstance, bufferSize, syncWrites);
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/storage/DataNodeFileManager.java b/src/main/java/com/actiontech/dble/memory/unsafe/storage/DataNodeFileManager.java
deleted file mode 100644
index 26cd571ad..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/storage/DataNodeFileManager.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.storage;
-
-
-import com.actiontech.dble.memory.unsafe.utils.JavaUtils;
-import com.actiontech.dble.memory.unsafe.utils.ServerPropertyConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-
-/**
- * Creates and maintains the logical mapping between logical blocks and physical on-disk
- * locations. One block is mapped to one file with a name given by its BlockId.
- *
- * Block files are hashed among the directories listed in local.dir
- */
-public class DataNodeFileManager {
- private static final Logger LOG = LoggerFactory.getLogger(DataNodeFileManager.class);
- private boolean deleteFilesOnStop;
- /**
- * TODO: delete tmp file
- */
- // The content of subDirs is immutable but the content of subDirs(i) is mutable. And the content
- // of subDirs(i) is protected by the lock of subDirs(i)
- // private val shutdownHook ;
- /* Create one local directory for each path mentioned in spark.local.dir; then, inside this
- * directory, create multiple subdirectories that we will hash files into, in order to avoid
- * having really large inodes at the top level. */
-
- private List
- * This class does not support concurrent writes. Also, once the writer has been opened it cannot be
- * reopened again.
- */
-public class DiskRowWriter extends OutputStream {
- /**
- * The file channel, used for repositioning / truncating the file.
- */
- private static final Logger LOG = LoggerFactory.getLogger(DiskRowWriter.class);
-
- private FileChannel channel = null;
- private OutputStream bs = null;
- private FileOutputStream fos = null;
- private SerializationStream objOut = null;
- private boolean initialized = false;
- private boolean hasBeenClosed = false;
-
- /**
- * Cursors used to represent positions in the file.
- *
- * xxxxxxxx|--------|--- |
- * ^ ^ ^
- * | | finalPosition
- * | reportedPosition
- * initialPosition
- *
- * initialPosition: Offset in the file where we start writing. Immutable.
- * reportedPosition: Position at the time of the last update to the write metrics.
- * finalPosition: Offset where we stopped writing. Set on closeAndCommit() then never changed.
- * -----: Current writes to the underlying file.
- * xxxxx: Existing contents of the file.
- */
- private long initialPosition = 0;
- private long reportedPosition = 0;
-
- /**
- * Keep track of number of records written and also use this to periodically
- * output bytes written since the latter is expensive to do for each record.
- */
- private long numRecordsWritten = 0;
-
- private File file;
- private SerializerInstance serializerInstance;
- private int bufferSize;
- private boolean syncWrites;
- // These write metrics concurrently shared with other active DiskBlockObjectWriters who
- // are themselves performing writes. All updates must be relative.
-
-
- public DiskRowWriter(
- File file,
- SerializerInstance serializerInstance,
- int bufferSize,
- boolean syncWrites) throws IOException {
-
- this.file = file;
- this.serializerInstance = serializerInstance;
- this.bufferSize = bufferSize;
- this.syncWrites = syncWrites;
- new FileOutputStream(file, false).close(); // for clean file
- /*
- ShuffleWriteMetrics writeMetrics,
- */
- initialPosition = file.length();
- reportedPosition = initialPosition;
- }
-
-
- public DiskRowWriter open() throws FileNotFoundException {
-
- if (hasBeenClosed) {
- throw new IllegalStateException("Writer already closed. Cannot be reopened.");
- }
-
- fos = new FileOutputStream(file, true);
- channel = fos.getChannel();
- bs = new BufferedOutputStream(fos, bufferSize);
- objOut = serializerInstance.serializeStream(bs);
- initialized = true;
-
- return this;
-
- }
-
-
- @Override
- public void close() {
- if (initialized) {
- try {
- if (syncWrites) {
- //Force outstanding writes to disk and track how long it takes
- objOut.flush();
- long start = System.nanoTime();
- fos.getFD().sync();
- // writeMetrics.incWriteTime(System.nanoTime() - start);
- }
- } catch (IOException e) {
- LOG.error(e.getMessage());
- } finally {
- objOut.close();
- }
- channel = null;
- bs = null;
- fos = null;
- objOut = null;
- initialized = false;
- hasBeenClosed = true;
- }
- }
-
- public boolean isOpen() {
- return objOut != null;
- }
-
- /**
- * Flush the partial writes and commit them as a single atomic block.
- */
- public void commitAndClose() throws IOException {
- if (initialized) {
- // NOTE: Because Kryo doesn’t flush the underlying stream we explicitly flush both the
- // serializer stream and the lower level stream.
- objOut.flush();
- bs.flush();
- close();
- }
- }
-
-
- /**
- * Reverts writes that haven’t been flushed yet. Callers should invoke this function
- * when there are runtime exceptions. This method will not throw, though it may be
- * unsuccessful in truncating written data.
- *
- * @return the file that this DiskRowWriter wrote to.
- */
- public File revertPartialWritesAndClose() throws IOException {
- // Discard current writes. We do this by flushing the outstanding writes and then
- // truncating the file to its initial position.
- try {
- if (initialized) {
- // writeMetrics.decBytesWritten(reportedPosition - initialPosition)
- // writeMetrics.decRecordsWritten(numRecordsWritten)
- objOut.flush();
- bs.flush();
- close();
- }
-
- try (FileOutputStream truncateStream = new FileOutputStream(file, true)) {
- truncateStream.getChannel().truncate(initialPosition);
- return file;
- }
- } catch (Exception e) {
- LOG.error(e.getMessage());
- return file;
- }
- }
-
-
- @Override
- public void write(int b) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void write(byte[] kvBytes, int offs, int len) throws IOException {
- if (!initialized) {
- open();
- }
-
- bs.write(kvBytes, offs, len);
- }
-
- /**
- * Notify the writer that a record worth of bytes has been written with OutputStream#write.
- */
- public void recordWritten() throws IOException {
- numRecordsWritten += 1;
- //writeMetrics.incRecordsWritten(1)
-
- //TODO: call updateBytesWritten() less frequently.
- if (numRecordsWritten % 32 == 0) {
- updateBytesWritten();
- }
- }
-
- /**
- * Report the number of bytes written in this writer’s shuffle write metrics.
- * Note that this is only valid before the underlying streams are closed.
- */
- private void updateBytesWritten() throws IOException {
- long pos = channel.position();
- //writeMetrics.incBytesWritten(pos - reportedPosition)
- reportedPosition = pos;
- }
-
- @Override
- public void flush() throws IOException {
- objOut.flush();
- bs.flush();
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/storage/DummySerializerInstance.java b/src/main/java/com/actiontech/dble/memory/unsafe/storage/DummySerializerInstance.java
deleted file mode 100644
index 60116b80f..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/storage/DummySerializerInstance.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.storage;
-
-
-import com.actiontech.dble.memory.unsafe.Platform;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-
-/**
- * Unfortunately, we need a serializer instance in order to construct a DiskRowWriter.
- * Our shuffle write path doesn't actually use this serializer (since we end up calling the
- * `write() OutputStream methods), but DiskRowWriter still calls some methods on it. To work
- * around this, we pass a dummy no-op serializer.
- */
-
-public final class DummySerializerInstance extends SerializerInstance {
-
- public static final DummySerializerInstance INSTANCE = new DummySerializerInstance();
-
- private DummySerializerInstance() {
- }
-
- @Override
- public SerializationStream serializeStream(final OutputStream s) {
- return new SerializationStream() {
- @Override
- public SerializationStream writeObject(Object o) {
- return null;
- }
-
- @Override
- public void flush() {
- // Need to implement this because DiskObjectWriter uses it to flush the compression stream
- try {
- s.flush();
- } catch (IOException e) {
- Platform.throwException(e);
- }
- }
-
- @Override
- public void close() {
- // Need to implement this because DiskObjectWriter uses it to close the compression stream
- try {
- s.close();
- } catch (IOException e) {
- Platform.throwException(e);
- }
- }
- };
- }
-
-
- public
- * First, we can leave out some units in interval string, and we only care about the value of
- * unit, so here we use non-capturing group to wrap the actual regex.
- * At the beginning of the actual regex, we should match spaces before the unit part.
- * Next is the number part, starts with an optional "-" to represent negative value. We use
- * capturing group to wrap this part as we need the value later.
- * Finally is the unit name, ends with an optional "s".
- */
- private static String unitRegex(String unit) {
- return "(?:\\s+(-?\\d+)\\s+" + unit + "s?)?";
- }
-
- private static Pattern p = Pattern.compile("interval" + unitRegex("year") + unitRegex("month") +
- unitRegex("week") + unitRegex("day") + unitRegex("hour") + unitRegex("minute") +
- unitRegex("second") + unitRegex("millisecond") + unitRegex("microsecond"));
-
- private static Pattern yearMonthPattern =
- Pattern.compile("^(?:['|\"])?([+|-])?(\\d+)-(\\d+)(?:['|\"])?$");
-
- private static Pattern dayTimePattern =
- Pattern.compile("^(?:['|\"])?([+|-])?(\\d+) (\\d+):(\\d+):(\\d+)(\\.(\\d+))?(?:['|\"])?$");
-
- private static Pattern quoteTrimPattern = Pattern.compile("^(?:['|\"])?(.*?)(?:['|\"])?$");
-
- private static long toLong(String s) {
- if (s == null) {
- return 0;
- } else {
- return Long.parseLong(s);
- }
- }
-
- public static CalendarInterval fromString(String s) {
- if (s == null) {
- return null;
- }
- s = s.trim();
- Matcher m = p.matcher(s);
- if (!m.matches() || s.equals("interval")) {
- return null;
- } else {
- long microseconds = toLong(m.group(3)) * MICROS_PER_WEEK;
- microseconds += toLong(m.group(4)) * MICROS_PER_DAY;
- microseconds += toLong(m.group(5)) * MICROS_PER_HOUR;
- microseconds += toLong(m.group(6)) * MICROS_PER_MINUTE;
- microseconds += toLong(m.group(7)) * MICROS_PER_SECOND;
- microseconds += toLong(m.group(8)) * MICROS_PER_MILLI;
- microseconds += toLong(m.group(9));
- long months = toLong(m.group(1)) * 12 + toLong(m.group(2));
- return new CalendarInterval((int) months, microseconds);
- }
- }
-
- public static long toLongWithRange(String fieldName,
- String s, long minValue, long maxValue) throws IllegalArgumentException {
- long result = 0;
- if (s != null) {
- result = Long.parseLong(s);
- if (result < minValue || result > maxValue) {
- throw new IllegalArgumentException(String.format("%s %d outside range [%d, %d]",
- fieldName, result, minValue, maxValue));
- }
- }
- return result;
- }
-
- /**
- * Parse YearMonth string in form: [-]YYYY-MM
- *
- * adapted from HiveIntervalYearMonth.valueOf
- */
- public static CalendarInterval fromYearMonthString(String s) throws IllegalArgumentException {
- CalendarInterval result = null;
- if (s == null) {
- throw new IllegalArgumentException("Interval year-month string was null");
- }
- s = s.trim();
- Matcher m = yearMonthPattern.matcher(s);
- if (!m.matches()) {
- throw new IllegalArgumentException(
- "Interval string does not match year-month format of 'y-m': " + s);
- } else {
- try {
- int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1;
- int years = (int) toLongWithRange("year", m.group(2), 0, Integer.MAX_VALUE);
- int months = (int) toLongWithRange("month", m.group(3), 0, 11);
- result = new CalendarInterval(sign * (years * 12 + months), 0);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Error parsing interval year-month string: " + e.getMessage(), e);
- }
- }
- return result;
- }
-
- /**
- * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn
- *
- * adapted from HiveIntervalDayTime.valueOf
- */
- public static CalendarInterval fromDayTimeString(String s) throws IllegalArgumentException {
- CalendarInterval result = null;
- if (s == null) {
- throw new IllegalArgumentException("Interval day-time string was null");
- }
- s = s.trim();
- Matcher m = dayTimePattern.matcher(s);
- if (!m.matches()) {
- throw new IllegalArgumentException(
- "Interval string does not match day-time format of 'd h:m:s.n': " + s);
- } else {
- try {
- int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1;
- long days = toLongWithRange("day", m.group(2), 0, Integer.MAX_VALUE);
- long hours = toLongWithRange("hour", m.group(3), 0, 23);
- long minutes = toLongWithRange("minute", m.group(4), 0, 59);
- long seconds = toLongWithRange("second", m.group(5), 0, 59);
- // Hive allow nanosecond precision interval
- long nanos = toLongWithRange("nanosecond", m.group(7), 0L, 999999999L);
- result = new CalendarInterval(0, sign * (
- days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes * MICROS_PER_MINUTE +
- seconds * MICROS_PER_SECOND + nanos / 1000L));
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Error parsing interval day-time string: " + e.getMessage(), e);
- }
- }
- return result;
- }
-
- public static CalendarInterval fromSingleUnitString(String unit, String s)
- throws IllegalArgumentException {
-
- CalendarInterval result = null;
- if (s == null) {
- throw new IllegalArgumentException(String.format("Interval %s string was null", unit));
- }
- s = s.trim();
- Matcher m = quoteTrimPattern.matcher(s);
- if (!m.matches()) {
- throw new IllegalArgumentException(
- "Interval string does not match day-time format of 'd h:m:s.n': " + s);
- } else {
- try {
- switch (unit) {
- case "year":
- int year = (int) toLongWithRange("year", m.group(1),
- Integer.MIN_VALUE / 12, Integer.MAX_VALUE / 12);
- result = new CalendarInterval(year * 12, 0L);
-
- break;
- case "month":
- int month = (int) toLongWithRange("month", m.group(1),
- Integer.MIN_VALUE, Integer.MAX_VALUE);
- result = new CalendarInterval(month, 0L);
-
- break;
- case "week":
- long week = toLongWithRange("week", m.group(1),
- Long.MIN_VALUE / MICROS_PER_WEEK, Long.MAX_VALUE / MICROS_PER_WEEK);
- result = new CalendarInterval(0, week * MICROS_PER_WEEK);
-
- break;
- case "day":
- long day = toLongWithRange("day", m.group(1),
- Long.MIN_VALUE / MICROS_PER_DAY, Long.MAX_VALUE / MICROS_PER_DAY);
- result = new CalendarInterval(0, day * MICROS_PER_DAY);
-
- break;
- case "hour":
- long hour = toLongWithRange("hour", m.group(1),
- Long.MIN_VALUE / MICROS_PER_HOUR, Long.MAX_VALUE / MICROS_PER_HOUR);
- result = new CalendarInterval(0, hour * MICROS_PER_HOUR);
-
- break;
- case "minute":
- long minute = toLongWithRange("minute", m.group(1),
- Long.MIN_VALUE / MICROS_PER_MINUTE, Long.MAX_VALUE / MICROS_PER_MINUTE);
- result = new CalendarInterval(0, minute * MICROS_PER_MINUTE);
-
- break;
- case "second": {
- long micros = parseSecondNano(m.group(1));
- result = new CalendarInterval(0, micros);
-
- break;
- }
- case "millisecond":
- long millisecond = toLongWithRange("millisecond", m.group(1),
- Long.MIN_VALUE / MICROS_PER_MILLI, Long.MAX_VALUE / MICROS_PER_MILLI);
- result = new CalendarInterval(0, millisecond * MICROS_PER_MILLI);
-
- break;
- case "microsecond": {
- long micros = Long.parseLong(m.group(1));
- result = new CalendarInterval(0, micros);
- break;
- }
- default:
- break;
- }
- } catch (Exception e) {
- throw new IllegalArgumentException("Error parsing interval string: " + e.getMessage(), e);
- }
- }
- return result;
- }
-
- /**
- * Parse second_nano string in ss.nnnnnnnnn format to microseconds
- */
- public static long parseSecondNano(String secondNano) throws IllegalArgumentException {
- String[] parts = secondNano.split("\\.");
- if (parts.length == 1) {
- return toLongWithRange("second", parts[0], Long.MIN_VALUE / MICROS_PER_SECOND,
- Long.MAX_VALUE / MICROS_PER_SECOND) * MICROS_PER_SECOND;
-
- } else if (parts.length == 2) {
- long seconds = parts[0].equals("") ? 0L : toLongWithRange("second", parts[0],
- Long.MIN_VALUE / MICROS_PER_SECOND, Long.MAX_VALUE / MICROS_PER_SECOND);
- long nanos = toLongWithRange("nanosecond", parts[1], 0L, 999999999L);
- return seconds * MICROS_PER_SECOND + nanos / 1000L;
-
- } else {
- throw new IllegalArgumentException(
- "Interval string does not match second-nano format of ss.nnnnnnnnn");
- }
- }
-
- public final int months;
- public final long microseconds;
-
- public CalendarInterval(int months, long microseconds) {
- this.months = months;
- this.microseconds = microseconds;
- }
-
- public CalendarInterval add(CalendarInterval that) {
- int newMonths = this.months + that.months;
- long newMicroseconds = this.microseconds + that.microseconds;
- return new CalendarInterval(newMonths, newMicroseconds);
- }
-
- public CalendarInterval subtract(CalendarInterval that) {
- int newMonths = this.months - that.months;
- long newMicroseconds = this.microseconds - that.microseconds;
- return new CalendarInterval(newMonths, newMicroseconds);
- }
-
- public CalendarInterval negate() {
- return new CalendarInterval(-this.months, -this.microseconds);
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) return true;
- if (other == null || !(other instanceof CalendarInterval)) return false;
-
- CalendarInterval o = (CalendarInterval) other;
- return this.months == o.months && this.microseconds == o.microseconds;
- }
-
- @Override
- public int hashCode() {
- return 31 * months + (int) microseconds;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("interval");
-
- if (months != 0) {
- appendUnit(sb, months / 12, "year");
- appendUnit(sb, months % 12, "month");
- }
-
- if (microseconds != 0) {
- long rest = microseconds;
- appendUnit(sb, rest / MICROS_PER_WEEK, "week");
- rest %= MICROS_PER_WEEK;
- appendUnit(sb, rest / MICROS_PER_DAY, "day");
- rest %= MICROS_PER_DAY;
- appendUnit(sb, rest / MICROS_PER_HOUR, "hour");
- rest %= MICROS_PER_HOUR;
- appendUnit(sb, rest / MICROS_PER_MINUTE, "minute");
- rest %= MICROS_PER_MINUTE;
- appendUnit(sb, rest / MICROS_PER_SECOND, "second");
- rest %= MICROS_PER_SECOND;
- appendUnit(sb, rest / MICROS_PER_MILLI, "millisecond");
- rest %= MICROS_PER_MILLI;
- appendUnit(sb, rest, "microsecond");
- }
-
- return sb.toString();
- }
-
- private void appendUnit(StringBuilder sb, long value, String unit) {
- if (value != 0) {
- sb.append(' ').append(value).append(' ').append(unit).append('s');
- }
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/types/UTF8String.java b/src/main/java/com/actiontech/dble/memory/unsafe/types/UTF8String.java
deleted file mode 100644
index a2295bbef..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/types/UTF8String.java
+++ /dev/null
@@ -1,1018 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.types;
-
-import com.actiontech.dble.memory.unsafe.Platform;
-import com.actiontech.dble.memory.unsafe.array.ByteArrayMethods;
-import com.actiontech.dble.memory.unsafe.hash.Murmur3OfX86And32Bit;
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.KryoSerializable;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-
-import javax.annotation.Nonnull;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Map;
-
-
-/**
- * A UTF-8 String for internal Spark use.
- *
- * A String encoded in UTF-8 as an Array[Byte], which can be used for comparison,
- * search, see http://en.wikipedia.org/wiki/UTF-8 for details.
- *
- * Note: This is not designed for general use cases, should not be used outside SQL.
- */
-public final class UTF8String implements Comparable
- * Note: `bytes` will be hold by returned UTF8String.
- */
- public static UTF8String fromBytes(byte[] bytes) {
- if (bytes != null) {
- return new UTF8String(bytes, Platform.BYTE_ARRAY_OFFSET, bytes.length);
- } else {
- return null;
- }
- }
-
- /**
- * Creates an UTF8String from byte array, which should be encoded in UTF-8.
- *
- * Note: `bytes` will be hold by returned UTF8String.
- */
- public static UTF8String fromBytes(byte[] bytes, int offset, int numBytes) {
- if (bytes != null) {
- return new UTF8String(bytes, Platform.BYTE_ARRAY_OFFSET + offset, numBytes);
- } else {
- return null;
- }
- }
-
- /**
- * Creates an UTF8String from given address (base and offset) and length.
- */
- public static UTF8String fromAddress(Object base, long offset, int numBytes) {
- return new UTF8String(base, offset, numBytes);
- }
-
- /**
- * Creates an UTF8String from String.
- */
- public static UTF8String fromString(String str) {
- return str == null ? null : fromBytes(str.getBytes(StandardCharsets.UTF_8));
- }
-
- /**
- * Creates an UTF8String that contains `length` spaces.
- */
- public static UTF8String blankString(int length) {
- byte[] spaces = new byte[length];
- Arrays.fill(spaces, (byte) ' ');
- return fromBytes(spaces);
- }
-
- protected UTF8String(Object base, long offset, int numBytes) {
- this.base = base;
- this.offset = offset;
- this.numBytes = numBytes;
- }
-
- // for serialization
- public UTF8String() {
- this(new Object(), 0, 0);
- }
-
- /**
- * Writes the content of this string into a memory address, identified by an object and an offset.
- * The target memory address must already been allocated, and have enough space to hold all the
- * bytes in this string.
- */
- public void writeToMemory(Object target, long targetOffset) {
- Platform.copyMemory(base, offset, target, targetOffset, numBytes);
- }
-
- public void writeTo(ByteBuffer buffer) {
- assert (buffer.hasArray());
- byte[] target = buffer.array();
- int arrayOffset = buffer.arrayOffset();
- int pos = buffer.position();
- writeToMemory(target, Platform.BYTE_ARRAY_OFFSET + arrayOffset + pos);
- buffer.position(pos + numBytes);
- }
-
- /**
- * Returns the number of bytes for a code point with the first byte as `b`
- *
- * @param b The first byte of a code point
- */
- private static int numBytesForFirstByte(final byte b) {
- final int offset = (b & 0xFF) - 192;
- return (offset >= 0) ? bytesOfCodePointInUTF8[offset] : 1;
- }
-
- /**
- * Returns the number of bytes
- */
- public int numBytes() {
- return numBytes;
- }
-
- /**
- * Returns the number of code points in it.
- */
- public int numChars() {
- int len = 0;
- for (int i = 0; i < numBytes; i += numBytesForFirstByte(getByte(i))) {
- len += 1;
- }
- return len;
- }
-
- /**
- * Returns a 64-bit integer that can be used as the prefix used in sorting.
- */
- public long getPrefix() {
- // Since JVMs are either 4-byte aligned or 8-byte aligned, we check the size of the string.
- // If size is 0, just return 0.
- // If size is between 0 and 4 (inclusive), assume data is 4-byte aligned under the hood and
- // use a getInt to fetch the prefix.
- // If size is greater than 4, assume we have at least 8 bytes of data to fetch.
- // After getting the data, we use a mask to mask out data that is not part of the string.
- long p;
- long mask = 0;
- if (isLittleEndian) {
- if (numBytes >= 8) {
- p = Platform.getLong(base, offset);
- } else if (numBytes > 4) {
- p = Platform.getLong(base, offset);
- mask = (1L << (8 - numBytes) * 8) - 1;
- } else if (numBytes > 0) {
- p = (long) Platform.getInt(base, offset);
- mask = (1L << (8 - numBytes) * 8) - 1;
- } else {
- p = 0;
- }
- p = Long.reverseBytes(p);
- } else {
- // byteOrder == ByteOrder.BIG_ENDIAN
- if (numBytes >= 8) {
- p = Platform.getLong(base, offset);
- } else if (numBytes > 4) {
- p = Platform.getLong(base, offset);
- mask = (1L << (8 - numBytes) * 8) - 1;
- } else if (numBytes > 0) {
- p = ((long) Platform.getInt(base, offset)) << 32;
- mask = (1L << (8 - numBytes) * 8) - 1;
- } else {
- p = 0;
- }
- }
- p &= ~mask;
- return p;
- }
-
- /**
- * Returns the underline bytes, will be a copy of it if it's part of another array.
- */
- public byte[] getBytes() {
- // avoid copy if `base` is `byte[]`
- if (offset == Platform.BYTE_ARRAY_OFFSET && base instanceof byte[] &&
- ((byte[]) base).length == numBytes) {
- return (byte[]) base;
- } else {
- byte[] bytes = new byte[numBytes];
- Platform.copyMemory(base, offset, bytes, Platform.BYTE_ARRAY_OFFSET, numBytes);
- return bytes;
- }
- }
-
- /**
- * Returns a substring of this.
- *
- * @param start the position of first code point
- * @param until the position after last code point, exclusive.
- */
- public UTF8String substring(final int start, final int until) {
- if (until <= start || start >= numBytes) {
- return EMPTY_UTF8;
- }
-
- int i = 0;
- int c = 0;
- while (i < numBytes && c < start) {
- i += numBytesForFirstByte(getByte(i));
- c += 1;
- }
-
- int j = i;
- while (i < numBytes && c < until) {
- i += numBytesForFirstByte(getByte(i));
- c += 1;
- }
-
- if (i > j) {
- byte[] bytes = new byte[i - j];
- Platform.copyMemory(base, offset + j, bytes, Platform.BYTE_ARRAY_OFFSET, i - j);
- return fromBytes(bytes);
- } else {
- return EMPTY_UTF8;
- }
- }
-
- public UTF8String substringSQL(int pos, int length) {
- // Information regarding the pos calculation:
- // Hive and SQL use one-based indexing for SUBSTR arguments but also accept zero and
- // negative indices for start positions. If a start index i is greater than 0, it
- // refers to element i-1 in the sequence. If a start index i is less than 0, it refers
- // to the -ith element before the end of the sequence. If a start index i is 0, it
- // refers to the first element.
- int len = numChars();
- int start = (pos > 0) ? pos - 1 : ((pos < 0) ? len + pos : 0);
- int end = (length == Integer.MAX_VALUE) ? len : start + length;
- return substring(start, end);
- }
-
- /**
- * Returns whether this contains `substring` or not.
- */
- public boolean contains(final UTF8String substring) {
- if (substring.numBytes == 0) {
- return true;
- }
-
- byte first = substring.getByte(0);
- for (int i = 0; i <= numBytes - substring.numBytes; i++) {
- if (getByte(i) == first && matchAt(substring, i)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Returns the byte at position `i`.
- */
- private byte getByte(int i) {
- return Platform.getByte(base, offset + i);
- }
-
- private boolean matchAt(final UTF8String s, int pos) {
- if (s.numBytes + pos > numBytes || pos < 0) {
- return false;
- }
- return ByteArrayMethods.arrayEquals(base, offset + pos, s.base, s.offset, s.numBytes);
- }
-
- public boolean startsWith(final UTF8String prefix) {
- return matchAt(prefix, 0);
- }
-
- public boolean endsWith(final UTF8String suffix) {
- return matchAt(suffix, numBytes - suffix.numBytes);
- }
-
- /**
- * Returns the upper case of this string
- */
- public UTF8String toUpperCase() {
- if (numBytes == 0) {
- return EMPTY_UTF8;
- }
-
- byte[] bytes = new byte[numBytes];
- bytes[0] = (byte) Character.toTitleCase(getByte(0));
- for (int i = 0; i < numBytes; i++) {
- byte b = getByte(i);
- if (numBytesForFirstByte(b) != 1) {
- // fallback
- return toUpperCaseSlow();
- }
- int upper = Character.toUpperCase((int) b);
- if (upper > 127) {
- // fallback
- return toUpperCaseSlow();
- }
- bytes[i] = (byte) upper;
- }
- return fromBytes(bytes);
- }
-
- private UTF8String toUpperCaseSlow() {
- return fromString(toString().toUpperCase());
- }
-
- /**
- * Returns the lower case of this string
- */
- public UTF8String toLowerCase() {
- if (numBytes == 0) {
- return EMPTY_UTF8;
- }
-
- byte[] bytes = new byte[numBytes];
- bytes[0] = (byte) Character.toTitleCase(getByte(0));
- for (int i = 0; i < numBytes; i++) {
- byte b = getByte(i);
- if (numBytesForFirstByte(b) != 1) {
- // fallback
- return toLowerCaseSlow();
- }
- int lower = Character.toLowerCase((int) b);
- if (lower > 127) {
- // fallback
- return toLowerCaseSlow();
- }
- bytes[i] = (byte) lower;
- }
- return fromBytes(bytes);
- }
-
- private UTF8String toLowerCaseSlow() {
- return fromString(toString().toLowerCase());
- }
-
- /**
- * Returns the title case of this string, that could be used as title.
- */
- public UTF8String toTitleCase() {
- if (numBytes == 0) {
- return EMPTY_UTF8;
- }
-
- byte[] bytes = new byte[numBytes];
- for (int i = 0; i < numBytes; i++) {
- byte b = getByte(i);
- if (i == 0 || getByte(i - 1) == ' ') {
- if (numBytesForFirstByte(b) != 1) {
- // fallback
- return toTitleCaseSlow();
- }
- int upper = Character.toTitleCase(b);
- if (upper > 127) {
- // fallback
- return toTitleCaseSlow();
- }
- bytes[i] = (byte) upper;
- } else {
- bytes[i] = b;
- }
- }
- return fromBytes(bytes);
- }
-
- private UTF8String toTitleCaseSlow() {
- StringBuilder sb = new StringBuilder();
- String s = toString();
- sb.append(s);
- sb.setCharAt(0, Character.toTitleCase(sb.charAt(0)));
- for (int i = 1; i < s.length(); i++) {
- if (sb.charAt(i - 1) == ' ') {
- sb.setCharAt(i, Character.toTitleCase(sb.charAt(i)));
- }
- }
- return fromString(sb.toString());
- }
-
- /*
- * Returns the index of the string `match` in this String. This string has to be a comma separated
- * list. If `match` contains a comma 0 will be returned. If the `match` isn't part of this String,
- * 0 will be returned, else the index of match (1-based index)
- */
- public int findInSet(UTF8String match) {
- if (match.contains(COMMA_UTF8)) {
- return 0;
- }
-
- int n = 1, lastComma = -1;
- for (int i = 0; i < numBytes; i++) {
- if (getByte(i) == (byte) ',') {
- if (i - (lastComma + 1) == match.numBytes &&
- ByteArrayMethods.arrayEquals(base, offset + (lastComma + 1), match.base, match.offset,
- match.numBytes)) {
- return n;
- }
- lastComma = i;
- n++;
- }
- }
- if (numBytes - (lastComma + 1) == match.numBytes &&
- ByteArrayMethods.arrayEquals(base, offset + (lastComma + 1), match.base, match.offset,
- match.numBytes)) {
- return n;
- }
- return 0;
- }
-
- /**
- * Copy the bytes from the current UTF8String, and make a new UTF8String.
- *
- * @param start the start position of the current UTF8String in bytes.
- * @param end the end position of the current UTF8String in bytes.
- * @return a new UTF8String in the position of [start, end] of current UTF8String bytes.
- */
- private UTF8String copyUTF8String(int start, int end) {
- int len = end - start + 1;
- byte[] newBytes = new byte[len];
- Platform.copyMemory(base, offset + start, newBytes, Platform.BYTE_ARRAY_OFFSET, len);
- return UTF8String.fromBytes(newBytes);
- }
-
- public UTF8String trim() {
- int s = 0;
- int e = this.numBytes - 1;
- // skip all of the space (0x20) in the left side
- while (s < this.numBytes && getByte(s) <= 0x20 && getByte(s) >= 0x00) s++;
- // skip all of the space (0x20) in the right side
- while (e >= 0 && getByte(e) <= 0x20 && getByte(e) >= 0x00) e--;
- if (s > e) {
- // empty string
- return UTF8String.fromBytes(new byte[0]);
- } else {
- return copyUTF8String(s, e);
- }
- }
-
- public UTF8String trimLeft() {
- int s = 0;
- // skip all of the space (0x20) in the left side
- while (s < this.numBytes && getByte(s) <= 0x20 && getByte(s) >= 0x00) s++;
- if (s == this.numBytes) {
- // empty string
- return UTF8String.fromBytes(new byte[0]);
- } else {
- return copyUTF8String(s, this.numBytes - 1);
- }
- }
-
- public UTF8String trimRight() {
- int e = numBytes - 1;
- // skip all of the space (0x20) in the right side
- while (e >= 0 && getByte(e) <= 0x20 && getByte(e) >= 0x00) e--;
-
- if (e < 0) {
- // empty string
- return UTF8String.fromBytes(new byte[0]);
- } else {
- return copyUTF8String(0, e);
- }
- }
-
- public UTF8String reverse() {
- byte[] result = new byte[this.numBytes];
-
- int i = 0; // position in byte
- while (i < numBytes) {
- int len = numBytesForFirstByte(getByte(i));
- Platform.copyMemory(this.base, this.offset + i, result,
- Platform.BYTE_ARRAY_OFFSET + result.length - i - len, len);
-
- i += len;
- }
-
- return UTF8String.fromBytes(result);
- }
-
- public UTF8String repeat(int times) {
- if (times <= 0) {
- return EMPTY_UTF8;
- }
-
- byte[] newBytes = new byte[numBytes * times];
- Platform.copyMemory(this.base, this.offset, newBytes, Platform.BYTE_ARRAY_OFFSET, numBytes);
-
- int copied = 1;
- while (copied < times) {
- int toCopy = Math.min(copied, times - copied);
- System.arraycopy(newBytes, 0, newBytes, copied * numBytes, numBytes * toCopy);
- copied += toCopy;
- }
-
- return UTF8String.fromBytes(newBytes);
- }
-
- /**
- * Returns the position of the first occurrence of substr in
- * current string from the specified position (0-based index).
- *
- * @param v the string to be searched
- * @param start the start position of the current string for searching
- * @return the position of the first occurrence of substr, if not found, -1 returned.
- */
- public int indexOf(UTF8String v, int start) {
- if (v.numBytes() == 0) {
- return 0;
- }
-
- // locate to the start position.
- int i = 0; // position in byte
- int c = 0; // position in character
- while (i < numBytes && c < start) {
- i += numBytesForFirstByte(getByte(i));
- c += 1;
- }
-
- do {
- if (i + v.numBytes > numBytes) {
- return -1;
- }
- if (ByteArrayMethods.arrayEquals(base, offset + i, v.base, v.offset, v.numBytes)) {
- return c;
- }
- i += numBytesForFirstByte(getByte(i));
- c += 1;
- } while (i < numBytes);
-
- return -1;
- }
-
- /**
- * Find the `str` from left to right.
- */
- private int find(UTF8String str, int start) {
- assert (str.numBytes > 0);
- while (start <= numBytes - str.numBytes) {
- if (ByteArrayMethods.arrayEquals(base, offset + start, str.base, str.offset, str.numBytes)) {
- return start;
- }
- start += 1;
- }
- return -1;
- }
-
- /**
- * Find the `str` from right to left.
- */
- private int rFind(UTF8String str, int start) {
- assert (str.numBytes > 0);
- while (start >= 0) {
- if (ByteArrayMethods.arrayEquals(base, offset + start, str.base, str.offset, str.numBytes)) {
- return start;
- }
- start -= 1;
- }
- return -1;
- }
-
- /**
- * Returns the substring from string str before count occurrences of the delimiter delim.
- * If count is positive, everything the left of the final delimiter (counting from left) is
- * returned. If count is negative, every to the right of the final delimiter (counting from the
- * right) is returned. subStringIndex performs a case-sensitive match when searching for delim.
- */
- public UTF8String subStringIndex(UTF8String delimiter, int count) {
- if (delimiter.numBytes == 0 || count == 0) {
- return EMPTY_UTF8;
- }
- if (count > 0) {
- int idx = -1;
- while (count > 0) {
- idx = find(delimiter, idx + 1);
- if (idx >= 0) {
- count--;
- } else {
- // can not find enough delim
- return this;
- }
- }
- if (idx == 0) {
- return EMPTY_UTF8;
- }
- byte[] bytes = new byte[idx];
- Platform.copyMemory(base, offset, bytes, Platform.BYTE_ARRAY_OFFSET, idx);
- return fromBytes(bytes);
-
- } else {
- int idx = numBytes - delimiter.numBytes + 1;
- count = -count;
- while (count > 0) {
- idx = rFind(delimiter, idx - 1);
- if (idx >= 0) {
- count--;
- } else {
- // can not find enough delim
- return this;
- }
- }
- if (idx + delimiter.numBytes == numBytes) {
- return EMPTY_UTF8;
- }
- int size = numBytes - delimiter.numBytes - idx;
- byte[] bytes = new byte[size];
- Platform.copyMemory(base, offset + idx + delimiter.numBytes, bytes, Platform.BYTE_ARRAY_OFFSET, size);
- return fromBytes(bytes);
- }
- }
-
- /**
- * Returns str, right-padded with pad to a length of len
- * For example:
- * ('hi', 5, '??') => 'hi???'
- * ('hi', 1, '??') => 'h'
- */
- public UTF8String rPad(int len, UTF8String pad) {
- int spaces = len - this.numChars(); // number of char need to pad
- if (spaces <= 0 || pad.numBytes() == 0) {
- // no padding at all, return the substring of the current string
- return substring(0, len);
- } else {
- int padChars = pad.numChars();
- int count = spaces / padChars; // how many padding string needed
- // the partial string of the padding
- UTF8String remain = pad.substring(0, spaces - padChars * count);
-
- byte[] data = new byte[this.numBytes + pad.numBytes * count + remain.numBytes];
- Platform.copyMemory(this.base, this.offset, data, Platform.BYTE_ARRAY_OFFSET, this.numBytes);
- int bytes = this.numBytes;
- int idx = 0;
- while (idx < count) {
- Platform.copyMemory(pad.base, pad.offset, data, Platform.BYTE_ARRAY_OFFSET + bytes, pad.numBytes);
- ++idx;
- bytes += pad.numBytes;
- }
- Platform.copyMemory(remain.base, remain.offset, data, Platform.BYTE_ARRAY_OFFSET + bytes, remain.numBytes);
-
- return UTF8String.fromBytes(data);
- }
- }
-
- /**
- * Returns str, left-padded with pad to a length of len.
- * For example:
- * ('hi', 5, '??') => '???hi'
- * ('hi', 1, '??') => 'h'
- */
- public UTF8String lPad(int len, UTF8String pad) {
- int spaces = len - this.numChars(); // number of char need to pad
- if (spaces <= 0 || pad.numBytes() == 0) {
- // no padding at all, return the substring of the current string
- return substring(0, len);
- } else {
- int padChars = pad.numChars();
- int count = spaces / padChars; // how many padding string needed
- // the partial string of the padding
- UTF8String remain = pad.substring(0, spaces - padChars * count);
-
- byte[] data = new byte[this.numBytes + pad.numBytes * count + remain.numBytes];
-
- int arrayOffset = 0;
- int idx = 0;
- while (idx < count) {
- Platform.copyMemory(pad.base, pad.offset, data, Platform.BYTE_ARRAY_OFFSET + arrayOffset, pad.numBytes);
- ++idx;
- arrayOffset += pad.numBytes;
- }
- Platform.copyMemory(remain.base, remain.offset, data, Platform.BYTE_ARRAY_OFFSET + arrayOffset, remain.numBytes);
- arrayOffset += remain.numBytes;
- Platform.copyMemory(this.base, this.offset, data, Platform.BYTE_ARRAY_OFFSET + arrayOffset, numBytes());
-
- return UTF8String.fromBytes(data);
- }
- }
-
- /**
- * Concatenates input strings together into a single string. Returns null if any input is null.
- */
- public static UTF8String concat(UTF8String... inputs) {
- // Compute the total length of the result.
- int totalLength = 0;
- for (UTF8String input1 : inputs) {
- if (input1 != null) {
- totalLength += input1.numBytes;
- } else {
- return null;
- }
- }
-
- // Allocate a new byte array, and copy the inputs one by one into it.
- final byte[] result = new byte[totalLength];
- int offset = 0;
- for (UTF8String input : inputs) {
- int len = input.numBytes;
- Platform.copyMemory(
- input.base, input.offset,
- result, Platform.BYTE_ARRAY_OFFSET + offset,
- len);
- offset += len;
- }
- return fromBytes(result);
- }
-
- /**
- * Concatenates input strings together into a single string using the separator.
- * A null input is skipped. For example, concat(",", "a", null, "c") would yield "a,c".
- */
- public static UTF8String concatWs(UTF8String separator, UTF8String... inputs) {
- if (separator == null) {
- return null;
- }
-
- int numInputBytes = 0; // total number of bytes from the inputs
- int numInputs = 0; // number of non-null inputs
- for (UTF8String input : inputs) {
- if (input != null) {
- numInputBytes += input.numBytes;
- numInputs++;
- }
- }
-
- if (numInputs == 0) {
- // Return an empty string if there is no input, or all the inputs are null.
- return fromBytes(new byte[0]);
- }
-
- // Allocate a new byte array, and copy the inputs one by one into it.
- // The size of the new array is the size of all inputs, plus the separators.
- final byte[] result = new byte[numInputBytes + (numInputs - 1) * separator.numBytes];
- int offset = 0;
-
- for (int i = 0, j = 0; i < inputs.length; i++) {
- if (inputs[i] != null) {
- int len = inputs[i].numBytes;
- Platform.copyMemory(
- inputs[i].base, inputs[i].offset,
- result, Platform.BYTE_ARRAY_OFFSET + offset,
- len);
- offset += len;
-
- j++;
- // Add separator if this is not the last input.
- if (j < numInputs) {
- Platform.copyMemory(
- separator.base, separator.offset,
- result, Platform.BYTE_ARRAY_OFFSET + offset,
- separator.numBytes);
- offset += separator.numBytes;
- }
- }
- }
- return fromBytes(result);
- }
-
- public UTF8String[] split(UTF8String pattern, int limit) {
- String[] splits = toString().split(pattern.toString(), limit);
- UTF8String[] res = new UTF8String[splits.length];
- for (int i = 0; i < res.length; i++) {
- res[i] = fromString(splits[i]);
- }
- return res;
- }
-
- // TODO: Need to use `Code Point` here instead of Char in case the character longer than 2 bytes
- public UTF8String translate(Map
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-import com.actiontech.dble.memory.unsafe.Platform;
-import com.google.common.annotations.VisibleForTesting;
-import sun.misc.Unsafe;
-
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-
-/**
- * Utility class that handles byte arrays, conversions to/from other types,
- */
-@SuppressWarnings("restriction")
-public final class BytesTools {
- private BytesTools() {
- }
-
- //HConstants.UTF8_ENCODING should be updated if this changed
- /**
- * When we encode strings, we always specify UTF8 encoding
- */
- private static final String UTF8_ENCODING = "UTF-8";
-
- //HConstants.UTF8_CHARSET should be updated if this changed
- /**
- * When we encode strings, we always specify UTF8 encoding
- */
- private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
-
- /**
- * Size of boolean in bytes
- */
- public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
-
- /**
- * Size of byte in bytes
- */
- public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
-
- /**
- * Size of char in bytes
- */
- public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
-
- /**
- * Size of double in bytes
- */
- public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
-
- /**
- * Size of float in bytes
- */
- public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
-
- /**
- * Size of int in bytes
- */
- public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
-
- /**
- * Size of long in bytes
- */
- public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
-
- /**
- * Size of short in bytes
- */
- public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
-
- /**
- * Convert a byte array to a int value
- *
- * @param buf
- * @return int
- * @throws NumberFormatException
- */
-
- public static int getInt(byte[] buf) throws NumberFormatException {
- return getInt(buf, 0, buf.length);
- }
-
- public static int getInt(byte[] buf, int offset, int endPos) throws NumberFormatException {
- byte base = 10;
-
- int s;
- for (s = offset; s < endPos && Character.isWhitespace((char) buf[s]); ++s) {
- //do nothing
- }
- if (s == endPos) {
- throw new NumberFormatException(toString(buf));
- } else {
- boolean negative = false;
- if ((char) buf[s] == 45) {
- negative = true;
- ++s;
- } else if ((char) buf[s] == 43) {
- ++s;
- }
-
- int save = s;
- int cutoff = 2147483647 / base;
- int cutLimit = 2147483647 % base;
- if (negative) {
- ++cutLimit;
- }
-
- boolean overflow = false;
-
- int i;
- for (i = 0; s < endPos; ++s) {
- char c = (char) buf[s];
- if (Character.isDigit(c)) {
- c = (char) (c - 48);
- } else {
- if (!Character.isLetter(c)) {
- break;
- }
-
- c = (char) (Character.toUpperCase(c) - 65 + 10);
- }
-
- if (c >= base) {
- break;
- }
-
- if (i <= cutoff && (i != cutoff || c <= cutLimit)) {
- i *= base;
- i += c;
- } else {
- overflow = true;
- }
- }
-
- if (s == save) {
- throw new NumberFormatException(toString(buf));
- } else if (overflow) {
- throw new NumberFormatException(toString(buf));
- } else {
- return negative ? -i : i;
- }
- }
- }
-
- /**
- * Convert a byte array to a long value
- *
- * @param buf
- * @return
- * @throws NumberFormatException
- */
- public static long getLong(byte[] buf) throws NumberFormatException {
- return getLong(buf, 0, buf.length);
- }
-
- public static long getLong(byte[] buf, int offset, int endPos) throws NumberFormatException {
- byte base = 10;
-
- int s;
- for (s = offset; s < endPos && Character.isWhitespace((char) buf[s]); ++s) {
- //do nothing
- }
-
- if (s == endPos) {
- throw new NumberFormatException(toString(buf));
- } else {
- boolean negative = false;
- if ((char) buf[s] == 45) {
- negative = true;
- ++s;
- } else if ((char) buf[s] == 43) {
- ++s;
- }
-
- int save = s;
- long cutoff = 9223372036854775807L / (long) base;
- long cutLimit = (long) ((int) (9223372036854775807L % (long) base));
- if (negative) {
- ++cutLimit;
- }
-
- boolean overflow = false;
-
- long i;
- for (i = 0L; s < endPos; ++s) {
- char c = (char) buf[s];
- if (Character.isDigit(c)) {
- c = (char) (c - 48);
- } else {
- if (!Character.isLetter(c)) {
- break;
- }
- c = (char) (Character.toUpperCase(c) - 65 + 10);
- }
-
- if (c >= base) {
- break;
- }
-
- if (i <= cutoff && (i != cutoff || (long) c <= cutLimit)) {
- i *= (long) base;
- i += (long) c;
- } else {
- overflow = true;
- }
- }
-
- if (s == save) {
- throw new NumberFormatException(toString(buf));
- } else if (overflow) {
- throw new NumberFormatException(toString(buf));
- } else {
- return negative ? -i : i;
- }
- }
- }
-
- /**
- * Convert a byte array to a short value
- *
- * @param buf
- * @return
- * @throws NumberFormatException
- */
- public static short getShort(byte[] buf) throws NumberFormatException {
- return getShort(buf, 0, buf.length);
- }
-
- public static short getShort(byte[] buf, int offset, int endPos) throws NumberFormatException {
- byte base = 10;
-
- int s;
- for (s = offset; s < endPos && Character.isWhitespace((char) buf[s]); ++s) {
- //do nothing
- }
-
- if (s == endPos) {
- throw new NumberFormatException(toString(buf));
- } else {
- boolean negative = false;
- if ((char) buf[s] == 45) {
- negative = true;
- ++s;
- } else if ((char) buf[s] == 43) {
- ++s;
- }
-
- int save = s;
- short cutoff = (short) (32767 / base);
- short cutLimit = (short) (32767 % base);
- if (negative) {
- ++cutLimit;
- }
-
- boolean overflow = false;
-
- short i;
- for (i = 0; s < endPos; ++s) {
- char c = (char) buf[s];
- if (Character.isDigit(c)) {
- c = (char) (c - 48);
- } else {
- if (!Character.isLetter(c)) {
- break;
- }
-
- c = (char) (Character.toUpperCase(c) - 65 + 10);
- }
-
- if (c >= base) {
- break;
- }
-
- if (i <= cutoff && (i != cutoff || c <= cutLimit)) {
- i = (short) (i * base);
- i = (short) (i + c);
- } else {
- overflow = true;
- }
- }
-
- if (s == save) {
- throw new NumberFormatException(toString(buf));
- } else if (overflow) {
- throw new NumberFormatException(toString(buf));
- } else {
- return negative ? (short) (-i) : i;
- }
- }
- }
-
- /**
- * Convert a byte array to a float value
- *
- * @param src
- * @return
- * @throws UnsupportedEncodingException
- */
- public static float getFloat(byte[] src) throws UnsupportedEncodingException {
- return Float.parseFloat(new String(src, "US-ASCII"));
- }
-
- /**
- * Convert a byte array to a double value
- *
- * @param src
- * @return
- * @throws UnsupportedEncodingException
- */
-
- public static double getDouble(byte[] src) throws UnsupportedEncodingException {
- return Double.parseDouble(new String(src, "US-ASCII"));
- }
-
- /**
- * Convert a long value to a byte array
- *
- * @param l
- * @return
- * @throws UnsupportedEncodingException
- */
-
-
- public static byte[] long2Bytes(long l) throws UnsupportedEncodingException {
- String lStr = Long.toString(l);
- return lStr.getBytes("US-ASCII");
- }
-
- /**
- * Convert a int value to a byte array
- *
- * @param i
- * @return
- * @throws UnsupportedEncodingException
- */
-
- public static byte[] int2Bytes(int i) throws UnsupportedEncodingException {
- String iStr = Integer.toString(i);
- return iStr.getBytes("US-ASCII");
- }
-
- /**
- * Convert a short value to a byte array
- *
- * @param i
- * @return
- * @throws UnsupportedEncodingException
- */
-
- public static byte[] short2Bytes(short i) throws UnsupportedEncodingException {
- String sStr = Short.toString(i);
- return sStr.getBytes("US-ASCII");
- }
-
- /**
- * Convert a float value to a byte array
- *
- * @param f
- * @return
- * @throws UnsupportedEncodingException
- */
- public static byte[] float2Bytes(float f) throws UnsupportedEncodingException {
- String fStr = Float.toString(f);
- return fStr.getBytes("US-ASCII");
- }
-
- /**
- * Convert a double value to a byte array
- *
- * @param d
- * @return
- * @throws UnsupportedEncodingException
- */
- public static byte[] double2Bytes(double d) throws UnsupportedEncodingException {
- String dStr = Double.toString(d);
- return dStr.getBytes("US-ASCII");
- }
-
- /**
- * Returns a new byte array, copied from the given {@code buf},
- * from the index 0 (inclusive) to the limit (exclusive),
- * regardless of the current position.
- * The position and the other index parameters are not changed.
- *
- * @param buf a byte buffer
- * @return the byte array
- */
- public static byte[] toBytes(ByteBuffer buf) {
- ByteBuffer dup = buf.duplicate();
- dup.position(0);
- return readBytes(dup);
- }
-
- /**
- * Converts a string to a UTF-8 byte array.
- *
- * @param s string
- * @return the byte array
- */
- public static byte[] toBytes(String s) {
- return s.getBytes(UTF8_CHARSET);
- }
-
- /**
- * Convert a boolean to a byte array. True becomes -1
- * and false becomes 0.
- *
- * @param b value
- * @return
- * Uses reflection to gracefully fall back to the Java implementation if
- * {@code Unsafe} isn't available.
- */
- @VisibleForTesting
- static class LexicographicalComparerHolder {
-
- static final String UNSAFE_COMPARER_NAME =
- LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
- static final Comparer
- * If no suffix is provided, the passed number is assumed to be in bytes.
- */
- public Long byteStringAsBytes(String str) {
- return JavaUtils.byteStringAsBytes(str);
- }
-
- /**
- * Convert a passed byte string (e.g. 50b, 100k, or 250m) to kibibytes for internal use.
- *
- * If no suffix is provided, the passed number is assumed to be in kibibytes.
- */
- public Long byteStringAsKb(String str) {
- return JavaUtils.byteStringAsKb(str);
- }
-
- /**
- * Convert a passed byte string (e.g. 50b, 100k, or 250m) to mebibytes for internal use.
- *
- * If no suffix is provided, the passed number is assumed to be in mebibytes.
- */
- public Long byteStringAsMb(String str) {
- return JavaUtils.byteStringAsMb(str);
- }
-
- /**
- * Convert a passed byte string (e.g. 50b, 100k, or 250m, 500g) to gibibytes for internal use.
- *
- * If no suffix is provided, the passed number is assumed to be in gibibytes.
- */
- public Long byteStringAsGb(String str) {
- return JavaUtils.byteStringAsGb(str);
- }
-
- /**
- * Convert a Java memory parameter passed to -Xmx (such as 300m or 1g) to a number of mebibytes.
- */
- public int memoryStringToMb(String str) {
- // Convert to bytes, rather than directly to MB, because when no units are specified the unit
- // is assumed to be bytes
- return (int) (JavaUtils.byteStringAsBytes(str) / 1024 / 1024);
- }
-
- public String getString(String s, String defaultValue) {
-
- String value = (String) settings.get(s);
- if (value != null) {
- return value;
- }
- return defaultValue;
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/utils/sort/AbstractScalaRowIterator.java b/src/main/java/com/actiontech/dble/memory/unsafe/utils/sort/AbstractScalaRowIterator.java
deleted file mode 100644
index a74019196..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/utils/sort/AbstractScalaRowIterator.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (C) 2016-2017 ActionTech.
- * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
- */
-
-package com.actiontech.dble.memory.unsafe.utils.sort;
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-/**
- * Created by zagnix 2016/6/6.
- */
-public class AbstractScalaRowIterator
- * Example format: an array of numbers, where each element is also the key.
- * See [[KVArraySortDataFormat]] for a more exciting format.
- *
- * Note: Declaring and instantiating multiple subclasses of this class would prevent JIT inlining
- * overridden methods and hence decrease the shuffle performance.
- *
- * @tparam K Type of the sort key of each element
- * @tparam Buffer Internal data structure used by a particular format (e.g., Array[Int]).
- */
-// TODO: Making Buffer a real trait would be a better abstraction, but adds some complexity.
-
-public abstract class SortDataFormat
- * The Java implementation is package private, and hence it cannot be called outside package
- */
-public class Sorter
- * This has been kept in Java with the original style in order to match very closely with the
- * Android source code, and thus be easy to verify correctness. The class is package private. We put
- * a simple Scala wrapper {@link Sorter}, which is available to
- * The purpose of the port is to generalize the interface to the sort to accept input data formats
- * besides simple arrays where every element is sorted individually. For instance, the AppendOnlyMap
- * uses this to sort an Array with alternating elements of the form [key, value, key, value].
- * This generalization comes with minimal overhead -- see SortDataFormat for more information.
- *
- * We allow key reuse to prevent creating many key objects -- see SortDataFormat.
- *
- * @see SortDataFormat
- * @see Sorter
- */
-class TimSort
- * This constant should be a power of two. It was 64 in Tim Peter's C
- * implementation, but 32 was empirically determined to work better in
- * this implementation. In the unlikely event that you set this constant
- * to be a number that's not a power of two, you'll need to change the
- * minRunLength computation.
- *
- * If you decrease this constant, you must change the stackLen
- * computation in the TimSort constructor, or you risk an
- * ArrayOutOfBounds exception. See listsort.txt for a discussion
- * of the minimum stack length required as a function of the length
- * of the array being sorted and the minimum merge sequence length.
- */
- private static final int MIN_MERGE = 32;
-
- private final SortDataFormat
- * This implementation was adapted from Tim Peters's list sort for
- * Python, which is described in detail here:
- *
- * http://svn.python.org/projects/python/trunk/Objects/listsort.txt
- *
- * Tim's C code may be found here:
- *
- * http://svn.python.org/projects/python/trunk/Objects/listobject.c
- *
- * The underlying techniques are described in this paper (and may have
- * even earlier origins):
- *
- * "Optimistic Sorting and Information Theoretic Complexity"
- * Peter McIlroy
- * SODA (Fourth Annual ACM-SIAM Symposium on Discrete Algorithms),
- * pp 467-474, Austin, Texas, 25-27 January 1993.
- *
- * While the API to this class consists solely of static methods, it is
- * (privately) instantiable; a TimSort instance holds the state of an ongoing
- * sort, assuming the input array is large enough to warrant the full-blown
- * TimSort. Small arrays are sorted in place, using a binary insertion sort.
- *
- * @author Josh Bloch
- */
- public void sort(B a, int lo, int hi, Comparator c) {
- assert c != null;
-
- int nRemaining = hi - lo;
- if (nRemaining < 2)
- return; // Arrays of size 0 and 1 are always sorted
-
- // If array is small, do a "mini-TimSort" with no merges
- if (nRemaining < MIN_MERGE) {
- int initRunLen = countRunAndMakeAscending(a, lo, hi, c);
- binarySort(a, lo, hi, lo + initRunLen, c);
- return;
- }
-
- /**
- * March over the array once, left to right, finding natural runs,
- * extending short natural runs to minRun elements, and merging runs
- * to maintain stack invariant.
- */
- SortState sortState = new SortState(a, c, hi - lo);
- int minRun = minRunLength(nRemaining);
- do {
- // Identify next run
- int runLen = countRunAndMakeAscending(a, lo, hi, c);
-
- // If run is short, extend to min(minRun, nRemaining)
- if (runLen < minRun) {
- int force = nRemaining <= minRun ? nRemaining : minRun;
- binarySort(a, lo, lo + force, lo + runLen, c);
- runLen = force;
- }
-
- // Push run onto pending-run stack, and maybe merge
- sortState.pushRun(lo, runLen);
- sortState.mergeCollapse();
-
- // Advance to find next run
- lo += runLen;
- nRemaining -= runLen;
- } while (nRemaining != 0);
-
- // Merge all remaining runs to complete sort
- assert lo == hi;
- sortState.mergeForceCollapse();
- assert sortState.stackSize == 1;
- }
-
- /**
- * Sorts the specified portion of the specified array using a binary
- * insertion sort. This is the best method for sorting small numbers
- * of elements. It requires O(n log n) compares, but O(n^2) data
- * movement (worst case).
- *
- * If the initial part of the specified range is already sorted,
- * this method can take advantage of it: the method assumes that the
- * elements from index {@code lo}, inclusive, to {@code start},
- * exclusive are already sorted.
- *
- * @param a the array in which a range is to be sorted
- * @param lo the index of the first element in the range to be sorted
- * @param hi the index after the last element in the range to be sorted
- * @param start the index of the first element in the range that is
- * not already known to be sorted ({@code lo <= start <= hi})
- * @param c comparator to used for the sort
- */
- @SuppressWarnings("fallthrough")
- private void binarySort(B a, int lo, int hi, int start, Comparator c) {
- assert lo <= start && start <= hi;
- if (start == lo)
- start++;
-
- K key0 = s.newKey();
- K key1 = s.newKey();
-
- B pivotStore = s.allocate(1);
- for (; start < hi; start++) {
- s.copyElement(a, start, pivotStore, 0);
- K pivot = s.getKey(pivotStore, 0, key0);
-
- // Set left (and right) to the index where a[start] (pivot) belongs
- int left = lo;
- int right = start;
- assert left <= right;
- /*
- * Invariants:
- * pivot >= all in [lo, left).
- * pivot < all in [right, start).
- */
- while (left < right) {
- int mid = (left + right) >>> 1;
- if (c.compare(pivot, s.getKey(a, mid, key1)) < 0)
- right = mid;
- else
- left = mid + 1;
- }
- assert left == right;
-
- /*
- * The invariants still hold: pivot >= all in [lo, left) and
- * pivot < all in [left, start), so pivot belongs at left. Note
- * that if there are elements equal to pivot, left points to the
- * first slot after them -- that's why this sort is stable.
- * Slide elements over to make room for pivot.
- */
- int n = start - left; // The number of elements to move
- // Switch is just an optimization for arraycopy in default case
- switch (n) {
- case 2:
- s.copyElement(a, left + 1, a, left + 2);
- s.copyElement(a, left, a, left + 1);
- break;
- case 1:
- s.copyElement(a, left, a, left + 1);
- break;
- default:
- s.copyRange(a, left, a, left + 1, n);
- }
- s.copyElement(pivotStore, 0, a, left);
- }
- }
-
- /**
- * Returns the length of the run beginning at the specified position in
- * the specified array and reverses the run if it is descending (ensuring
- * that the run will always be ascending when the method returns).
- *
- * A run is the longest ascending sequence with:
- *
- * a[lo] <= a[lo + 1] <= a[lo + 2] <= ...
- *
- * or the longest descending sequence with:
- *
- * a[lo] > a[lo + 1] > a[lo + 2] > ...
- *
- * For its intended use in a stable mergesort, the strictness of the
- * definition of "descending" is needed so that the call can safely
- * reverse a descending sequence without violating stability.
- *
- * @param a the array in which a run is to be counted and possibly reversed
- * @param lo index of the first element in the run
- * @param hi index after the last element that may be contained in the run.
- * It is required that {@code lo < hi}.
- * @param c the comparator to used for the sort
- * @return the length of the run beginning at the specified position in
- * the specified array
- */
- private int countRunAndMakeAscending(B a, int lo, int hi, Comparator c) {
- assert lo < hi;
- int runHi = lo + 1;
- if (runHi == hi)
- return 1;
-
- K key0 = s.newKey();
- K key1 = s.newKey();
-
- // Find end of run, and reverse range if descending
- if (c.compare(s.getKey(a, runHi++, key0), s.getKey(a, lo, key1)) < 0) { // Descending
- while (runHi < hi && c.compare(s.getKey(a, runHi, key0), s.getKey(a, runHi - 1, key1)) < 0)
- runHi++;
- reverseRange(a, lo, runHi);
- } else { // Ascending
- while (runHi < hi && c.compare(s.getKey(a, runHi, key0), s.getKey(a, runHi - 1, key1)) >= 0)
- runHi++;
- }
-
- return runHi - lo;
- }
-
- /**
- * Reverse the specified range of the specified array.
- *
- * @param a the array in which a range is to be reversed
- * @param lo the index of the first element in the range to be reversed
- * @param hi the index after the last element in the range to be reversed
- */
- private void reverseRange(B a, int lo, int hi) {
- hi--;
- while (lo < hi) {
- s.swap(a, lo, hi);
- lo++;
- hi--;
- }
- }
-
- /**
- * Returns the minimum acceptable run length for an array of the specified
- * length. Natural runs shorter than this will be extended with
- * {@link #binarySort}.
- *
- * Roughly speaking, the computation is:
- *
- * If n < MIN_MERGE, return n (it's too small to bother with fancy stuff).
- * Else if n is an exact power of 2, return MIN_MERGE/2.
- * Else return an int k, MIN_MERGE/2 <= k <= MIN_MERGE, such that n/k
- * is close to, but strictly less than, an exact power of 2.
- *
- * For the rationale, see listsort.txt.
- *
- * @param n the length of the array to be sorted
- * @return the length of the minimum run to be merged
- */
- private int minRunLength(int n) {
- assert n >= 0;
- int r = 0; // Becomes 1 if any 1 bits are shifted off
- while (n >= MIN_MERGE) {
- r |= (n & 1);
- n >>= 1;
- }
- return n + r;
- }
-
- private final class SortState {
-
- /**
- * The Buffer being sorted.
- */
- private final B a;
-
- /**
- * Length of the sort Buffer.
- */
- private final int aLength;
-
- /**
- * The comparator for this sort.
- */
- private final Comparator c;
-
- /**
- * When we get into galloping mode, we stay there until both runs win less
- * often than MIN_GALLOP consecutive times.
- */
- private static final int MIN_GALLOP = 7;
-
- /**
- * This controls when we get *into* galloping mode. It is initialized
- * to MIN_GALLOP. The mergeLo and mergeHi methods nudge it higher for
- * random data, and lower for highly structured data.
- */
- private int minGallop = MIN_GALLOP;
-
- /**
- * Maximum initial size of tmp array, which is used for merging. The array
- * can grow to accommodate demand.
- *
- * Unlike Tim's original C version, we do not allocate this much storage
- * when sorting smaller arrays. This change was required for performance.
- */
- private static final int INITIAL_TMP_STORAGE_LENGTH = 256;
-
- /**
- * Temp storage for merges.
- */
- private B tmp; // Actual runtime type will be Object[], regardless of T
-
- /**
- * Length of the temp storage.
- */
- private int tmpLength = 0;
-
- /**
- * A stack of pending runs yet to be merged. Run i starts at
- * address base[i] and extends for len[i] elements. It's always
- * true (so long as the indices are in bounds) that:
- *
- * runBase[i] + runLen[i] == runBase[i + 1]
- *
- * so we could cut the storage for this, but it's a minor amount,
- * and keeping all the info explicit simplifies the code.
- */
- private int stackSize = 0; // Number of pending runs on stack
- private final int[] runBase;
- private final int[] runLen;
-
- /**
- * Creates a TimSort instance to maintain the state of an ongoing sort.
- *
- * @param a the array to be sorted
- * @param c the comparator to determine the order of the sort
- */
- private SortState(B a, Comparator c, int len) {
- this.aLength = len;
- this.a = a;
- this.c = c;
-
- // Allocate temp storage (which may be increased later if necessary)
- tmpLength = len < 2 * INITIAL_TMP_STORAGE_LENGTH ? len >>> 1 : INITIAL_TMP_STORAGE_LENGTH;
- tmp = s.allocate(tmpLength);
-
- /*
- * Allocate runs-to-be-merged stack (which cannot be expanded). The
- * stack length requirements are described in listsort.txt. The C
- * version always uses the same stack length (85), but this was
- * measured to be too expensive when sorting "mid-sized" arrays (e.g.,
- * 100 elements) in Java. Therefore, we use smaller (but sufficiently
- * large) stack lengths for smaller arrays. The "magic numbers" in the
- * computation below must be changed if MIN_MERGE is decreased. See
- * the MIN_MERGE declaration above for more information.
- */
- int stackLen = (len < 120 ? 5 :
- len < 1542 ? 10 :
- len < 119151 ? 19 : 40);
- runBase = new int[stackLen];
- runLen = new int[stackLen];
- }
-
- /**
- * Pushes the specified run onto the pending-run stack.
- *
- * @param base index of the first element in the run
- * @param len the number of elements in the run
- */
- private void pushRun(int base, int len) {
- this.runBase[stackSize] = base;
- this.runLen[stackSize] = len;
- stackSize++;
- }
-
- /**
- * Examines the stack of runs waiting to be merged and merges adjacent runs
- * until the stack invariants are reestablished:
- *
- * 1. runLen[i - 3] > runLen[i - 2] + runLen[i - 1]
- * 2. runLen[i - 2] > runLen[i - 1]
- *
- * This method is called each time a new run is pushed onto the stack,
- * so the invariants are guaranteed to hold for i < stackSize upon
- * entry to the method.
- */
- private void mergeCollapse() {
- while (stackSize > 1) {
- int n = stackSize - 2;
- if ((n >= 1 && runLen[n - 1] <= runLen[n] + runLen[n + 1]) ||
- (n >= 2 && runLen[n - 2] <= runLen[n] + runLen[n - 1])) {
- if (runLen[n - 1] < runLen[n + 1])
- n--;
- } else if (runLen[n] > runLen[n + 1]) {
- break; // Invariant is established
- }
- mergeAt(n);
- }
- }
-
- /**
- * Merges all runs on the stack until only one remains. This method is
- * called once, to complete the sort.
- */
- private void mergeForceCollapse() {
- while (stackSize > 1) {
- int n = stackSize - 2;
- if (n > 0 && runLen[n - 1] < runLen[n + 1])
- n--;
- mergeAt(n);
- }
- }
-
- /**
- * Merges the two runs at stack indices i and i+1. Run i must be
- * the penultimate or antepenultimate run on the stack. In other words,
- * i must be equal to stackSize-2 or stackSize-3.
- *
- * @param i stack index of the first of the two runs to merge
- */
- private void mergeAt(int i) {
- assert stackSize >= 2;
- assert i >= 0;
- assert i == stackSize - 2 || i == stackSize - 3;
-
- int base1 = runBase[i];
- int len1 = runLen[i];
- int base2 = runBase[i + 1];
- int len2 = runLen[i + 1];
- assert len1 > 0 && len2 > 0;
- assert base1 + len1 == base2;
-
- /*
- * Record the length of the combined runs; if i is the 3rd-last
- * run now, also slide over the last run (which isn't involved
- * in this merge). The current run (i+1) goes away in any case.
- */
- runLen[i] = len1 + len2;
- if (i == stackSize - 3) {
- runBase[i + 1] = runBase[i + 2];
- runLen[i + 1] = runLen[i + 2];
- }
- stackSize--;
-
- K key0 = s.newKey();
-
- /*
- * Find where the first element of run2 goes in run1. Prior elements
- * in run1 can be ignored (because they're already in place).
- */
- int k = gallopRight(s.getKey(a, base2, key0), a, base1, len1, 0, c);
- assert k >= 0;
- base1 += k;
- len1 -= k;
- if (len1 == 0)
- return;
-
- /*
- * Find where the last element of run1 goes in run2. Subsequent elements
- * in run2 can be ignored (because they're already in place).
- */
- len2 = gallopLeft(s.getKey(a, base1 + len1 - 1, key0), a, base2, len2, len2 - 1, c);
- assert len2 >= 0;
- if (len2 == 0)
- return;
-
- // Merge remaining runs, using tmp array with min(len1, len2) elements
- if (len1 <= len2)
- mergeLo(base1, len1, base2, len2);
- else
- mergeHi(base1, len1, base2, len2);
- }
-
- /**
- * Locates the position at which to insert the specified key into the
- * specified sorted range; if the range contains an element equal to key,
- * returns the index of the leftmost equal element.
- *
- * @param key the key whose insertion point to search for
- * @param search the array in which to search
- * @param base the index of the first element in the range
- * @param len the length of the range; must be > 0
- * @param hint the index at which to begin the search, 0 <= hint < n.
- * The closer hint is to the result, the faster this method will run.
- * @param comparator the comparator used to order the range, and to search
- * @return the int k, 0 <= k <= n such that a[b + k - 1] < key <= a[b + k],
- * pretending that a[b - 1] is minus infinity and a[b + n] is infinity.
- * In other words, key belongs at index b + k; or in other words,
- * the first k elements of a should precede key, and the last n - k
- * should follow it.
- */
- private int gallopLeft(K key, B search, int base, int len, int hint, Comparator comparator) {
- assert len > 0 && hint >= 0 && hint < len;
- int lastOfs = 0;
- int ofs = 1;
- K key0 = s.newKey();
-
- if (comparator.compare(key, s.getKey(search, base + hint, key0)) > 0) {
- // Gallop right until a[base+hint+lastOfs] < key <= a[base+hint+ofs]
- int maxOfs = len - hint;
- while (ofs < maxOfs && comparator.compare(key, s.getKey(search, base + hint + ofs, key0)) > 0) {
- lastOfs = ofs;
- ofs = (ofs << 1) + 1;
- if (ofs <= 0) // int overflow
- ofs = maxOfs;
- }
- if (ofs > maxOfs)
- ofs = maxOfs;
-
- // Make offsets relative to base
- lastOfs += hint;
- ofs += hint;
- } else { // key <= a[base + hint]
- // Gallop left until a[base+hint-ofs] < key <= a[base+hint-lastOfs]
- final int maxOfs = hint + 1;
- while (ofs < maxOfs && comparator.compare(key, s.getKey(search, base + hint - ofs, key0)) <= 0) {
- lastOfs = ofs;
- ofs = (ofs << 1) + 1;
- if (ofs <= 0) // int overflow
- ofs = maxOfs;
- }
- if (ofs > maxOfs)
- ofs = maxOfs;
-
- // Make offsets relative to base
- int tmpValue = lastOfs;
- lastOfs = hint - ofs;
- ofs = hint - tmpValue;
- }
- assert -1 <= lastOfs && lastOfs < ofs && ofs <= len;
-
- /*
- * Now a[base+lastOfs] < key <= a[base+ofs], so key belongs somewhere
- * to the right of lastOfs but no farther right than ofs. Do a binary
- * search, with invariant a[base + lastOfs - 1] < key <= a[base + ofs].
- */
- lastOfs++;
- while (lastOfs < ofs) {
- int m = lastOfs + ((ofs - lastOfs) >>> 1);
-
- if (comparator.compare(key, s.getKey(search, base + m, key0)) > 0)
- lastOfs = m + 1; // a[base + m] < key
- else
- ofs = m; // key <= a[base + m]
- }
- assert lastOfs == ofs; // so a[base + ofs - 1] < key <= a[base + ofs]
- return ofs;
- }
-
- /**
- * Like gallopLeft, except that if the range contains an element equal to
- * key, gallopRight returns the index after the rightmost equal element.
- *
- * @param key the key whose insertion point to search for
- * @param search the array in which to search
- * @param base the index of the first element in the range
- * @param len the length of the range; must be > 0
- * @param hint the index at which to begin the search, 0 <= hint < n.
- * The closer hint is to the result, the faster this method will run.
- * @param comparator the comparator used to order the range, and to search
- * @return the int k, 0 <= k <= n such that a[b + k - 1] <= key < a[b + k]
- */
- private int gallopRight(K key, B search, int base, int len, int hint, Comparator comparator) {
- assert len > 0 && hint >= 0 && hint < len;
-
- int ofs = 1;
- int lastOfs = 0;
- K key1 = s.newKey();
-
- if (comparator.compare(key, s.getKey(search, base + hint, key1)) < 0) {
- // Gallop left until a[b+hint - ofs] <= key < a[b+hint - lastOfs]
- int maxOfs = hint + 1;
- while (ofs < maxOfs && comparator.compare(key, s.getKey(search, base + hint - ofs, key1)) < 0) {
- lastOfs = ofs;
- ofs = (ofs << 1) + 1;
- if (ofs <= 0) // int overflow
- ofs = maxOfs;
- }
- if (ofs > maxOfs)
- ofs = maxOfs;
-
- // Make offsets relative to b
- int tmpValue = lastOfs;
- lastOfs = hint - ofs;
- ofs = hint - tmpValue;
- } else { // a[b + hint] <= key
- // Gallop right until a[b+hint + lastOfs] <= key < a[b+hint + ofs]
- int maxOfs = len - hint;
- while (ofs < maxOfs && comparator.compare(key, s.getKey(search, base + hint + ofs, key1)) >= 0) {
- lastOfs = ofs;
- ofs = (ofs << 1) + 1;
- if (ofs <= 0) // int overflow
- ofs = maxOfs;
- }
- if (ofs > maxOfs)
- ofs = maxOfs;
-
- // Make offsets relative to b
- lastOfs += hint;
- ofs += hint;
- }
- assert -1 <= lastOfs && lastOfs < ofs && ofs <= len;
-
- /*
- * Now a[b + lastOfs] <= key < a[b + ofs], so key belongs somewhere to
- * the right of lastOfs but no farther right than ofs. Do a binary
- * search, with invariant a[b + lastOfs - 1] <= key < a[b + ofs].
- */
- lastOfs++;
- while (lastOfs < ofs) {
- int m = lastOfs + ((ofs - lastOfs) >>> 1);
-
- if (comparator.compare(key, s.getKey(search, base + m, key1)) < 0)
- ofs = m; // key < a[b + m]
- else
- lastOfs = m + 1; // a[b + m] <= key
- }
- assert lastOfs == ofs; // so a[b + ofs - 1] <= key < a[b + ofs]
- return ofs;
- }
-
- /**
- * Merges two adjacent runs in place, in a stable fashion. The first
- * element of the first run must be greater than the first element of the
- * second run (a[base1] > a[base2]), and the last element of the first run
- * (a[base1 + len1-1]) must be greater than all elements of the second run.
- *
- * For performance, this method should be called only when len1 <= len2;
- * its twin, mergeHi should be called if len1 >= len2. (Either method
- * may be called if len1 == len2.)
- *
- * @param base1 index of first element in first run to be merged
- * @param len1 length of first run to be merged (must be > 0)
- * @param base2 index of first element in second run to be merged
- * (must be aBase + aLen)
- * @param len2 length of second run to be merged (must be > 0)
- */
- private void mergeLo(int base1, int len1, int base2, int len2) {
- assert len1 > 0 && len2 > 0 && base1 + len1 == base2;
-
- // Copy first run into temp array
- B src = this.a; // For performance
- B dst = ensureCapacity(len1);
- s.copyRange(src, base1, dst, 0, len1);
-
- int cursor1 = 0; // Indexes into tmp array
- int cursor2 = base2; // Indexes int a
- int dst2 = base1; // Indexes int a
-
- // Move first element of second run and deal with degenerate cases
- s.copyElement(src, cursor2++, src, dst2++);
- if (--len2 == 0) {
- s.copyRange(dst, cursor1, src, dst2, len1);
- return;
- }
- if (len1 == 1) {
- s.copyRange(src, cursor2, src, dst2, len2);
- s.copyElement(dst, cursor1, src, dst2 + len2); // Last elt of run 1 to end of merge
- return;
- }
-
- K key0 = s.newKey();
- K key1 = s.newKey();
-
- Comparator comparator = this.c; // Use local variable for performance
- int gallop = this.minGallop; // " " " " "
- outer:
- while (true) {
- int count1 = 0; // Number of times in a row that first run won
- int count2 = 0; // Number of times in a row that second run won
-
- /*
- * Do the straightforward thing until (if ever) one run starts
- * winning consistently.
- */
- do {
- assert len1 > 1 && len2 > 0;
- if (comparator.compare(s.getKey(src, cursor2, key0), s.getKey(dst, cursor1, key1)) < 0) {
- s.copyElement(src, cursor2++, src, dst2++);
- count2++;
- count1 = 0;
- if (--len2 == 0)
- break outer;
- } else {
- s.copyElement(dst, cursor1++, src, dst2++);
- count1++;
- count2 = 0;
- if (--len1 == 1)
- break outer;
- }
- } while ((count1 | count2) < gallop);
-
- /*
- * One run is winning so consistently that galloping may be a
- * huge win. So try that, and continue galloping until (if ever)
- * neither run appears to be winning consistently anymore.
- */
- do {
- assert len1 > 1 && len2 > 0;
- count1 = gallopRight(s.getKey(src, cursor2, key0), dst, cursor1, len1, 0, comparator);
- if (count1 != 0) {
- s.copyRange(dst, cursor1, src, dst2, count1);
- dst2 += count1;
- cursor1 += count1;
- len1 -= count1;
- if (len1 <= 1) // len1 == 1 || len1 == 0
- break outer;
- }
- s.copyElement(src, cursor2++, src, dst2++);
- if (--len2 == 0)
- break outer;
-
- count2 = gallopLeft(s.getKey(dst, cursor1, key0), src, cursor2, len2, 0, comparator);
- if (count2 != 0) {
- s.copyRange(src, cursor2, src, dst2, count2);
- dst2 += count2;
- cursor2 += count2;
- len2 -= count2;
- if (len2 == 0)
- break outer;
- }
- s.copyElement(dst, cursor1++, src, dst2++);
- if (--len1 == 1)
- break outer;
- gallop--;
- } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP);
- if (gallop < 0)
- gallop = 0;
- gallop += 2; // Penalize for leaving gallop mode
- } // End of "outer" loop
- this.minGallop = gallop < 1 ? 1 : gallop; // Write back to field
-
- if (len1 == 1) {
- assert len2 > 0;
- s.copyRange(src, cursor2, src, dst2, len2);
- s.copyElement(dst, cursor1, src, dst2 + len2); // Last elt of run 1 to end of merge
- } else if (len1 == 0) {
- throw new IllegalArgumentException(
- "Comparison method violates its general contract!");
- } else {
- assert len2 == 0;
- assert len1 > 1;
- s.copyRange(dst, cursor1, src, dst2, len1);
- }
- }
-
- /**
- * Like mergeLo, except that this method should be called only if
- * len1 >= len2; mergeLo should be called if len1 <= len2. (Either method
- * may be called if len1 == len2.)
- *
- * @param base1 index of first element in first run to be merged
- * @param len1 length of first run to be merged (must be > 0)
- * @param base2 index of first element in second run to be merged
- * (must be aBase + aLen)
- * @param len2 length of second run to be merged (must be > 0)
- */
- private void mergeHi(int base1, int len1, int base2, int len2) {
- assert len1 > 0 && len2 > 0 && base1 + len1 == base2;
-
- // Copy second run into temp array
- B src = this.a; // For performance
- B dst = ensureCapacity(len2);
- s.copyRange(src, base2, dst, 0, len2);
-
- int cursor1 = base1 + len1 - 1; // Indexes into a
- int cursor2 = len2 - 1; // Indexes into tmp array
- int dst2 = base2 + len2 - 1; // Indexes into a
-
- K key0 = s.newKey();
- K key1 = s.newKey();
-
- // Move last element of first run and deal with degenerate cases
- s.copyElement(src, cursor1--, src, dst2--);
- if (--len1 == 0) {
- s.copyRange(dst, 0, src, dst2 - (len2 - 1), len2);
- return;
- }
- if (len2 == 1) {
- dst2 -= len1;
- cursor1 -= len1;
- s.copyRange(src, cursor1 + 1, src, dst2 + 1, len1);
- s.copyElement(dst, cursor2, src, dst2);
- return;
- }
-
- Comparator comparator = this.c; // Use local variable for performance
- int gallop = this.minGallop; // " " " " "
- outer:
- while (true) {
- int count1 = 0; // Number of times in a row that first run won
- int count2 = 0; // Number of times in a row that second run won
-
- /*
- * Do the straightforward thing until (if ever) one run
- * appears to win consistently.
- */
- do {
- assert len1 > 0 && len2 > 1;
- if (comparator.compare(s.getKey(dst, cursor2, key0), s.getKey(src, cursor1, key1)) < 0) {
- s.copyElement(src, cursor1--, src, dst2--);
- count1++;
- count2 = 0;
- if (--len1 == 0)
- break outer;
- } else {
- s.copyElement(dst, cursor2--, src, dst2--);
- count2++;
- count1 = 0;
- if (--len2 == 1)
- break outer;
- }
- } while ((count1 | count2) < gallop);
-
- /*
- * One run is winning so consistently that galloping may be a
- * huge win. So try that, and continue galloping until (if ever)
- * neither run appears to be winning consistently anymore.
- */
- do {
- assert len1 > 0 && len2 > 1;
- count1 = len1 - gallopRight(s.getKey(dst, cursor2, key0), src, base1, len1, len1 - 1, comparator);
- if (count1 != 0) {
- dst2 -= count1;
- cursor1 -= count1;
- len1 -= count1;
- s.copyRange(src, cursor1 + 1, src, dst2 + 1, count1);
- if (len1 == 0)
- break outer;
- }
- s.copyElement(dst, cursor2--, src, dst2--);
- if (--len2 == 1)
- break outer;
-
- count2 = len2 - gallopLeft(s.getKey(src, cursor1, key0), dst, 0, len2, len2 - 1, comparator);
- if (count2 != 0) {
- dst2 -= count2;
- cursor2 -= count2;
- len2 -= count2;
- s.copyRange(dst, cursor2 + 1, src, dst2 + 1, count2);
- if (len2 <= 1) // len2 == 1 || len2 == 0
- break outer;
- }
- s.copyElement(src, cursor1--, src, dst2--);
- if (--len1 == 0)
- break outer;
- gallop--;
- } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP);
- if (gallop < 0)
- gallop = 0;
- gallop += 2; // Penalize for leaving gallop mode
- } // End of "outer" loop
- this.minGallop = gallop < 1 ? 1 : gallop; // Write back to field
-
- if (len2 == 1) {
- assert len1 > 0;
- dst2 -= len1;
- cursor1 -= len1;
- s.copyRange(src, cursor1 + 1, src, dst2 + 1, len1);
- s.copyElement(dst, cursor2, src, dst2); // Move first elt of run2 to front of merge
- } else if (len2 == 0) {
- throw new IllegalArgumentException(
- "Comparison method violates its general contract!");
- } else {
- assert len1 == 0;
- assert len2 > 0;
- s.copyRange(dst, 0, src, dst2 - (len2 - 1), len2);
- }
- }
-
- /**
- * Ensures that the external array tmp has at least the specified
- * number of elements, increasing its size if necessary. The size
- * increases exponentially to ensure amortized linear time complexity.
- *
- * @param minCapacity the minimum required capacity of the tmp array
- * @return tmp, whether or not it grew
- */
- private B ensureCapacity(int minCapacity) {
- if (tmpLength < minCapacity) {
- // Compute smallest power of 2 > minCapacity
- int newSize = minCapacity;
- newSize |= newSize >> 1;
- newSize |= newSize >> 2;
- newSize |= newSize >> 4;
- newSize |= newSize >> 8;
- newSize |= newSize >> 16;
- newSize++;
-
- if (newSize < 0) // Not bloody likely!
- newSize = minCapacity;
- else
- newSize = Math.min(newSize, aLength >>> 1);
-
- tmp = s.allocate(newSize);
- tmpLength = newSize;
- }
- return tmp;
- }
- }
-}
diff --git a/src/main/java/com/actiontech/dble/memory/unsafe/utils/sort/UnsafeExternalRowSorter.java b/src/main/java/com/actiontech/dble/memory/unsafe/utils/sort/UnsafeExternalRowSorter.java
deleted file mode 100644
index 590994f73..000000000
--- a/src/main/java/com/actiontech/dble/memory/unsafe/utils/sort/UnsafeExternalRowSorter.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.actiontech.dble.memory.unsafe.utils.sort;
-
-import com.actiontech.dble.memory.SeverMemory;
-import com.actiontech.dble.memory.unsafe.Platform;
-import com.actiontech.dble.memory.unsafe.memory.mm.DataNodeMemoryManager;
-import com.actiontech.dble.memory.unsafe.row.StructType;
-import com.actiontech.dble.memory.unsafe.row.UnsafeRow;
-import com.actiontech.dble.sqlengine.mpp.OrderCol;
-import com.actiontech.dble.sqlengine.mpp.RowDataPacketSorter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nonnull;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.Iterator;
-import java.util.List;
-
-public final class UnsafeExternalRowSorter {
-
- private final Logger logger = LoggerFactory.getLogger(UnsafeExternalRowSorter.class);
-
- private long numRowsInserted = 0;
- private final StructType schema;
- private final PrefixComputer prefixComputer;
- private final UnsafeExternalSorter sorter;
- private final PrefixComparator prefixComparator;
- private final RecordComparator recordComparator;
-
-
- public UnsafeExternalRowSorter(DataNodeMemoryManager dataNodeMemoryManager,
- @Nonnull SeverMemory memory,
- StructType schema,
- PrefixComparator prefixComparator,
- PrefixComputer prefixComputer,
- long pageSizeBytes,
- boolean canUseRadixSort,
- boolean enableSort) throws IOException {
- this.schema = schema;
- this.prefixComputer = prefixComputer;
- this.prefixComparator = prefixComparator;
- this.recordComparator = new RowComparator(schema);
- sorter = UnsafeExternalSorter.create(
- dataNodeMemoryManager,
- memory.getBlockManager(),
- memory.getSerializerManager(),
- recordComparator,
- prefixComparator,
- memory.getConf().getSizeAsBytes("server.pointer.array.len", "16K"),
- pageSizeBytes,
- canUseRadixSort,
- enableSort);
- }
-
-
- public void insertRow(UnsafeRow row) throws IOException {
- final long prefix = prefixComputer.computePrefix(row);
-
- sorter.insertRecord(
- row.getBaseObject(),
- row.getBaseOffset(),
- row.getSizeInBytes(),
- prefix);
-
- numRowsInserted++;
- }
-
- /**
- * Return total rows
- */
- public long getNumRowsInserted() {
- return numRowsInserted;
- }
-
- /**
- * Return the peak memory used so far, in bytes.
- */
- public long getPeakMemoryUsage() {
- return sorter.getPeakMemoryUsedBytes();
- }
-
- /**
- * @return the total amount of time spent sorting data (in-memory only).
- */
- public long getSortTimeNanos() {
- return sorter.getSortTimeNanos();
- }
-
- public void cleanupResources() {
- logger.debug("row sorter clean up resources!!!");
- sorter.cleanupResources();
- }
-
- public Iterator
- * record length (4 bytes), key length (4 bytes), key data, value data
- *
- * record length = key length + value length + 4
- */
- public void insertKVRecord(Object keyBase, long keyOffset, int keyLen,
- Object valueBase, long valueOffset, int valueLen, long prefix)
- throws IOException {
-
- growPointerArrayIfNecessary();
- final int required = keyLen + valueLen + 4 + 4;
- acquireNewPageIfNecessary(required);
-
- /**
- * insert k-v into currentPage(MemoryBlock),the position is pageCursor
- */
- final Object base = currentPage.getBaseObject();
- /**
- * record address according to currentPage and pageCursor
- */
- final long recordAddress = dataNodeMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor);
-
- /**
- * total length of a row =keyLen + valueLen + record length (4 for int)
- */
- Platform.putInt(base, pageCursor, keyLen + valueLen + 4/**record length*/);
-
- /**
- * add 4 bytes
- */
- pageCursor += 4;
- /**
- * the size of key len
- */
- Platform.putInt(base, pageCursor, keyLen);
-
- /**
- * add 4 bytes
- */
- pageCursor += 4;
-
- /**
- * record the key
- */
- Platform.copyMemory(keyBase, keyOffset, base, pageCursor, keyLen);
- /**
- * add keyLen bytes
- */
- pageCursor += keyLen;
-
- /**
- * record the value
- */
- Platform.copyMemory(valueBase, valueOffset, base, pageCursor, valueLen);
-
- /**
- * add valueLen bytes
- */
- pageCursor += valueLen;
-
- assert (inMemSorter != null);
- /**
- * insert the pointer into the longArray
- * longArray point to the real value of the pointer in the Page
- */
- inMemSorter.insertRecord(recordAddress, prefix);
- }
-
- /**
- * Merges another UnsafeExternalSorters into this one, the other one will be emptied.
- *
- * @throws IOException
- */
- public void merge(UnsafeExternalSorter other) throws IOException {
- other.spill();
- spillWriters.addAll(other.spillWriters);
- // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`.
- other.spillWriters.clear();
- other.cleanupResources();
- }
-
- /**
- * SpillableIterator is an Iterator for order in memory /disk?
- * Returns a sorted iterator. It is the caller's responsibility to call `cleanupResources()`
- * after consuming this iterator.
- */
-
- public UnsafeSorterIterator getSortedIterator() throws IOException {
- assert (recordComparator != null);
- if (spillWriters.isEmpty()) {
- assert (inMemSorter != null);
- readingIterator = new SpillableIterator(inMemSorter.getSortedIterator());
- return readingIterator;
- } else {
- /**
- * merger files of UnsafeSorterSpillWriter and order it
- */
- final UnsafeSorterSpillMerger spillMerger =
- new UnsafeSorterSpillMerger(recordComparator, prefixComparator, spillWriters.size());
-
- for (UnsafeSorterSpillWriter spillWriter : spillWriters) {
- /**
- * add to UnsafeSorterSpillMerger from UnsafeSorterSpillReader
- */
- spillMerger.addSpillIfNotEmpty(spillWriter.getReader(serializerManager));
- }
- if (inMemSorter != null) {
- readingIterator = new SpillableIterator(inMemSorter.getSortedIterator());
- spillMerger.addSpillIfNotEmpty(readingIterator);
- }
- /**
- * sort
- */
- return spillMerger.getSortedIterator();
- }
- }
-
- /**
- * Returns a iterator, which will return the rows in the order as inserted.
- *
- * It is the caller's responsibility to call `cleanupResources()`
- * after consuming this iterator.
- *
- * TODO: support forced spilling
- */
- public UnsafeSorterIterator getIterator() throws IOException {
- /**
- * if spillWriters is empty,read from memory
- */
- if (spillWriters.isEmpty()) {
- assert (inMemSorter != null);
- return inMemSorter.getSortedIterator();
- } else {
- /**
- * read data from file of spillWriters to UnsafeSorterIterator
- * and add to the queue
- */
- LinkedList
- * Within each long[] buffer, position {@code 2 * i} holds a pointer pointer to the record at
- * index {@code i}, while position {@code 2 * i + 1} in the array holds an 8-byte key prefix.
- */
-public final class UnsafeSortDataFormat
- extends SortDataFormat
- * [# of records (int)] [[len (int)][prefix (long)][data (bytes)]...]
- */
-public final class UnsafeSorterSpillWriter {
-
- static final int DISK_WRITE_BUFFER_SIZE = 1024 * 1024;
-
- // Small writes to DiskRowWriter will be fairly inefficient. Since there doesn't seem to
- // be an API to directly transfer bytes from managed memory to the disk writer, we buffer
- // data through a byte array.
- private byte[] writeBuffer = new byte[DISK_WRITE_BUFFER_SIZE];
-
- private final File file;
- private final ConnectionId conId;
- private final int numRecordsToWrite;
- private DiskRowWriter writer;
- private int numRecordsSpilled = 0;
-
- public UnsafeSorterSpillWriter(
- DataNodeDiskManager blockManager,
- int fileBufferSize,
- int numRecordsToWrite) throws IOException {
-
- DataNodeFileManager diskBlockManager = blockManager.diskBlockManager();
- this.conId = diskBlockManager.createTempLocalBlock();
- this.file = diskBlockManager.getFile(this.conId);
-
- this.numRecordsToWrite = numRecordsToWrite;
- // Unfortunately, we need a serializer instance in order to construct a DiskRowWriter.
- // Our write path doesn't actually use this serializer (since we end up calling the `write()`
- // OutputStream methods), but DiskRowWriter still calls some methods on it. To work
- // around this, we pass a dummy no-op serializer.
- writer = blockManager.getDiskWriter(file, DummySerializerInstance.INSTANCE, fileBufferSize/**,writeMetrics*/);
- // Write the number of records
- writeIntToBuffer(numRecordsToWrite, 0);
- writer.write(writeBuffer, 0, 4);
- }
-
- // Based on DataOutputStream.writeLong.
- private void writeLongToBuffer(long v, int offset) throws IOException {
- writeBuffer[offset + 0] = (byte) (v >>> 56);
- writeBuffer[offset + 1] = (byte) (v >>> 48);
- writeBuffer[offset + 2] = (byte) (v >>> 40);
- writeBuffer[offset + 3] = (byte) (v >>> 32);
- writeBuffer[offset + 4] = (byte) (v >>> 24);
- writeBuffer[offset + 5] = (byte) (v >>> 16);
- writeBuffer[offset + 6] = (byte) (v >>> 8);
- writeBuffer[offset + 7] = (byte) (v >>> 0);
- }
-
- // Based on DataOutputStream.writeInt.
- private void writeIntToBuffer(int v, int offset) throws IOException {
- writeBuffer[offset + 0] = (byte) (v >>> 24);
- writeBuffer[offset + 1] = (byte) (v >>> 16);
- writeBuffer[offset + 2] = (byte) (v >>> 8);
- writeBuffer[offset + 3] = (byte) (v >>> 0);
- }
-
- /**
- * Write a record to a spill file.
- *
- * @param baseObject the base object / memory page containing the record
- * @param baseOffset the base offset which points directly to the record data.
- * @param recordLength the length of the record.
- * @param keyPrefix a sort key prefix
- */
- public void write(
- Object baseObject,
- long baseOffset,
- int recordLength,
- long keyPrefix) throws IOException {
- if (numRecordsSpilled == numRecordsToWrite) {
- throw new IllegalStateException(
- "Number of records written exceeded numRecordsToWrite = " + numRecordsToWrite);
- } else {
- numRecordsSpilled++;
- }
-
- /**
- * [# of records (int)] [[len (int)][prefix (long)][data (bytes)]...]
- * a row format
- * */
-
- /**
- * recordLength 4bytes
- */
- writeIntToBuffer(recordLength, 0);
- /**
- * key,8bytes
- */
- writeLongToBuffer(keyPrefix, 4);
- /**
- * dataRemaining real data bytes
- */
- int dataRemaining = recordLength;
- /**
- * freeSpace
- */
- int freeSpaceInWriteBuffer = DISK_WRITE_BUFFER_SIZE - 4 - 8; // space used by prefix + len
-
- /**
- *recordReadPosition
- */
- long recordReadPosition = baseOffset;
-
- while (dataRemaining > 0) {
- /**
- * read real data ,min(freeSpaceInWriteBuffer,dataRemaining)
- */
- final int toTransfer = Math.min(freeSpaceInWriteBuffer, dataRemaining);
-
- /**
- * copy from baseObjectto writeBuffer
- */
- Platform.copyMemory(
- baseObject, /**srd*/
- recordReadPosition, /**offset*/
- writeBuffer, /**write dst*/
- Platform.BYTE_ARRAY_OFFSET + (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer), /**write offset*/
- toTransfer);
-
- /**
- * write writeBufferto disk
- */
- writer.write(writeBuffer, 0, (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer) + toTransfer);
- /**
- * add toTransfer size
- */
- recordReadPosition += toTransfer;
- /**
- * calc dataRemainingrecord
- */
- dataRemaining -= toTransfer;
- /**
- * init the WriteBuffer to DISK_WRITE_BUFFER_SIZE
- */
- freeSpaceInWriteBuffer = DISK_WRITE_BUFFER_SIZE;
- }
-
- /**
- * write the remain data to disk
- */
- if (freeSpaceInWriteBuffer < DISK_WRITE_BUFFER_SIZE) {
-
- writer.write(writeBuffer, 0, (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer));
-
- }
-
- writer.recordWritten();
- }
-
- public void close() throws IOException {
- writer.commitAndClose();
- writer = null;
- writeBuffer = null;
- }
-
- public File getFile() {
- return file;
- }
-
- public UnsafeSorterSpillReader getReader(SerializerManager serializerManager) throws IOException {
- return new UnsafeSorterSpillReader(serializerManager, file, conId);
- }
-}
diff --git a/src/main/java/com/actiontech/dble/net/mysql/BinaryRowDataPacket.java b/src/main/java/com/actiontech/dble/net/mysql/BinaryRowDataPacket.java
index 3dccbbcbc..58fa1b599 100644
--- a/src/main/java/com/actiontech/dble/net/mysql/BinaryRowDataPacket.java
+++ b/src/main/java/com/actiontech/dble/net/mysql/BinaryRowDataPacket.java
@@ -8,7 +8,6 @@ package com.actiontech.dble.net.mysql;
import com.actiontech.dble.backend.mysql.BufferUtil;
import com.actiontech.dble.config.Fields;
-import com.actiontech.dble.memory.unsafe.row.UnsafeRow;
import com.actiontech.dble.net.FrontendConnection;
import com.actiontech.dble.util.ByteUtil;
import com.actiontech.dble.util.DateUtil;
@@ -50,34 +49,6 @@ public class BinaryRowDataPacket extends MySQLPacket {
public BinaryRowDataPacket() {
}
- /**
- * transform from UnsafeRow to BinaryRowDataPacket
- *
- * Notice: when isOffHeapuseOffHeapForMergeis enable,
- * UnsafeRow package the data,
- * so now need to unpackage to BinaryRowDataPacket
- *
- * @param fields
- * @param unsafeRow
- */
- public void read(List
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.actiontech.dble.memory.unsafe.sort;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * This codes generates a int array which fails the standard TimSort.
- *
- * The blog that reported the bug
- * http://www.envisage-project.eu/timsort-specification-and-verification/
- *
- * This codes was originally wrote by Stijn de Gouw, modified by Evan Yu to adapt to
- * our test suite.
- *
- * https://github.com/abstools/java-timsort-bug
- * https://github.com/abstools/java-timsort-bug/blob/master/LICENSE
- */
-public class TestTimSort {
-
- private static final int MIN_MERGE = 32;
-
- /**
- * Returns an array of integers that demonstrate the bug in TimSort
- */
- public static int[] getTimSortBugTestSet(int length) {
- int minRun = minRunLength(length);
- List
- * Location loc = map.lookup(keyBase, keyOffset, keyLength);
- * if (!loc.isDefined()) {
- * if (!loc.append(keyBase, keyOffset, keyLength, ...)) {
- * // handle failure to grow map (by spilling, for example)
- * }
- * }
- *
- * b encoded in a byte array.
- */
- public static byte[] toBytes(final boolean b) {
- return new byte[]{b ? (byte) -1 : (byte) 0};
- }
-
- /**
- * Convert a long value to a byte array using big-endian.
- *
- * @param val value to convert
- * @return the byte array
- */
- public static byte[] toBytes(long val) {
- byte[] b = new byte[8];
- for (int i = 7; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- private static byte[] readBytes(ByteBuffer buf) {
- byte[] result = new byte[buf.remaining()];
- buf.get(result);
- return result;
- }
-
- /**
- * @param b Presumed UTF-8 encoded byte array.
- * @return String made from b
- */
- public static String toString(final byte[] b) {
- if (b == null) {
- return null;
- }
- return toString(b, 0, b.length);
- }
-
- /**
- * Joins two byte arrays together using a separator.
- *
- * @param b1 The first byte array.
- * @param sep The separator to use.
- * @param b2 The second byte array.
- */
- public static String toString(final byte[] b1,
- String sep,
- final byte[] b2) {
- return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
- }
-
- /**
- * This method will convert utf8 encoded bytes into a string. If
- * the given byte array is null, this method will return null.
- *
- * @param b Presumed UTF-8 encoded byte array.
- * @param off offset into array
- * @return String made from b or null
- */
- public static String toString(final byte[] b, int off) {
- if (b == null) {
- return null;
- }
- int len = b.length - off;
- if (len <= 0) {
- return "";
- }
- return new String(b, off, len, UTF8_CHARSET);
- }
-
- /**
- * This method will convert utf8 encoded bytes into a string. If
- * the given byte array is null, this method will return null.
- *
- * @param b Presumed UTF-8 encoded byte array.
- * @param off offset into array
- * @param len length of utf-8 sequence
- * @return String made from b or null
- */
- public static String toString(final byte[] b, int off, int len) {
- if (b == null) {
- return null;
- }
- if (len == 0) {
- return "";
- }
- return new String(b, off, len, UTF8_CHARSET);
- }
-
- /**
- * Write a printable representation of a byte array.
- *
- * @param b byte array
- * @return string
- * @see #toStringBinary(byte[], int, int)
- */
- public static String toStringBinary(final byte[] b) {
- if (b == null)
- return "null";
- return toStringBinary(b, 0, b.length);
- }
-
- /**
- * Converts the given byte buffer to a printable representation,
- * from the index 0 (inclusive) to the limit (exclusive),
- * regardless of the current position.
- * The position and the other index parameters are not changed.
- *
- * @param buf a byte buffer
- * @return a string representation of the buffer's binary contents
- * @see #toBytes(ByteBuffer)
- */
- public static String toStringBinary(ByteBuffer buf) {
- if (buf == null)
- return "null";
- if (buf.hasArray()) {
- return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
- }
- return toStringBinary(toBytes(buf));
- }
-
- private static final char[] HEX_CHARS_UPPER = {
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F',
- };
-
- /**
- * Write a printable representation of a byte array. Non-printable
- * characters are hex escaped in the format \\x%02X, eg:
- * \x00 \x05 etc
- *
- * @param b array to write out
- * @param off offset to start at
- * @param len length to write
- * @return string output
- */
- public static String toStringBinary(final byte[] b, int off, int len) {
- StringBuilder result = new StringBuilder();
- // Just in case we are passed a 'len' that is > buffer length...
- if (off >= b.length) return result.toString();
- if (off + len > b.length) len = b.length - off;
- for (int i = off; i < off + len; ++i) {
- int ch = b[i] & 0xFF;
- if (ch >= ' ' && ch <= '~' && ch != '\\') {
- result.append((char) ch);
- } else {
- result.append("\\x");
- result.append(HEX_CHARS_UPPER[ch / 0x10]);
- result.append(HEX_CHARS_UPPER[ch % 0x10]);
- }
- }
- return result.toString();
- }
-
- private static boolean isHexDigit(char c) {
- return
- (c >= 'A' && c <= 'F') ||
- (c >= '0' && c <= '9');
- }
-
- /**
- * Takes a ASCII digit in the range A-F0-9 and returns
- * the corresponding integer/ordinal value.
- *
- * @param ch The hex digit.
- * @return The converted hex value as a byte.
- */
- public static byte toBinaryFromHex(byte ch) {
- if (ch >= 'A' && ch <= 'F')
- return (byte) ((byte) 10 + (byte) (ch - 'A'));
- // else
- return (byte) (ch - '0');
- }
-
- public static byte[] toBytesBinary(String in) {
- // this may be bigger than we need, but let's be safe.
- byte[] b = new byte[in.length()];
- int size = 0;
- for (int i = 0; i < in.length(); ++i) {
- char ch = in.charAt(i);
- if (ch == '\\' && in.length() > i + 1 && in.charAt(i + 1) == 'x') {
- // ok, take next 2 hex digits.
- char hd1 = in.charAt(i + 2);
- char hd2 = in.charAt(i + 3);
-
- // they need to be A-F0-9:
- if (!isHexDigit(hd1) ||
- !isHexDigit(hd2)) {
- // bogus escape code, ignore:
- continue;
- }
- // turn hex ASCII digit -> number
- byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2));
-
- b[size++] = d;
- i += 3; // skip 3
- } else {
- b[size++] = (byte) ch;
- }
- }
- // resize:
- byte[] b2 = new byte[size];
- System.arraycopy(b, 0, b2, 0, size);
- return b2;
- }
-
- /**
- * Reverses {@link #toBytes(boolean)}
- *
- * @param b array
- * @return True or false.
- */
- public static boolean toBoolean(final byte[] b) {
- if (b.length != 1) {
- throw new IllegalArgumentException("Array has wrong size: " + b.length);
- }
- return b[0] != (byte) 0;
- }
-
- /**
- * @param left left operand
- * @param right right operand
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(final byte[] left, final byte[] right) {
- return LexicographicalComparerHolder.BEST_COMPARER.
- compareTo(left, 0, left.length, right, 0, right.length);
- }
-
- /**
- * Lexicographically compare two arrays.
- *
- * @param buffer1 left operand
- * @param buffer2 right operand
- * @param offset1 Where to start comparing in the left buffer
- * @param offset2 Where to start comparing in the right buffer
- * @param length1 How much to compare from the left buffer
- * @param length2 How much to compare from the right buffer
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(byte[] buffer1, int offset1, int length1,
- byte[] buffer2, int offset2, int length2) {
- return LexicographicalComparerHolder.BEST_COMPARER.
- compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
- }
-
- @VisibleForTesting
- static Comparercolumn
- */
- public static byte[][] toByteArrays(final String column) {
- return toByteArrays(toBytes(column));
- }
-
- /**
- * @param column operand
- * @return A byte array of a byte array where first and only entry is
- * column
- */
- public static byte[][] toByteArrays(final byte[] column) {
- byte[][] result = new byte[1][];
- result[0] = column;
- return result;
- }
-
- /**
- * @param t operands
- * @return Array of binary byte arrays made from passed array of binary strings
- */
- public static byte[][] toBinaryByteArrays(final String[] t) {
- byte[][] result = new byte[t.length][];
- for (int i = 0; i < t.length; i++) {
- result[i] = BytesTools.toBytesBinary(t[i]);
- }
- return result;
- }
-
-
- public static byte[] paddingInt(byte[] a) {
-
- if (a == null) {
- return null;
- }
-
- if (a.length == SIZEOF_INT) {
- return a;
- }
-
- byte[] b = new byte[SIZEOF_INT];
- if (Platform.LITTLE_ENDIAN) {
- for (int i = 0; i < SIZEOF_INT - a.length; i++) {
- b[i] = 0x00;
- }
- System.arraycopy(a, 0, b, SIZEOF_INT - a.length, a.length);
- } else {
- System.arraycopy(a, 0, b, 0, a.length);
- for (int i = a.length; i < SIZEOF_INT; i++) {
- b[i] = 0x00;
- }
- }
- return b;
- }
-
- public static byte[] paddingLong(byte[] a) {
- if (a == null) {
- return null;
- }
-
- if (a.length == SIZEOF_LONG) {
- return a;
- }
-
- byte[] b = new byte[SIZEOF_LONG];
- if (Platform.LITTLE_ENDIAN) {
- for (int i = 0; i < SIZEOF_LONG - a.length; i++) {
- b[i] = 0x00;
- }
- System.arraycopy(a, 0, b, SIZEOF_LONG - a.length, a.length);
- } else {
- System.arraycopy(a, 0, b, 0, a.length);
- for (int i = a.length; i < SIZEOF_LONG; i++) {
- b[i] = 0x00;
- }
- }
- return b;
- }
-
- public static byte[] paddingShort(byte[] a) {
-
- if (a == null) {
- return null;
- }
-
- if (a.length == SIZEOF_SHORT) {
- return a;
- }
- byte[] b = new byte[SIZEOF_SHORT];
- if (Platform.LITTLE_ENDIAN) {
- for (int i = 0; i < SIZEOF_SHORT - a.length; i++) {
- b[i] = 0x00;
- }
- System.arraycopy(a, 0, b, SIZEOF_SHORT - a.length, a.length);
- } else {
- System.arraycopy(a, 0, b, 0, a.length);
- for (int i = a.length; i < SIZEOF_SHORT; i++) {
- b[i] = 0x00;
- }
- }
- return b;
- }
-
- /**
- * Split passed range. Expensive operation relatively. Uses BigInteger math.
- * Useful splitting ranges for MapReduce jobs.
- *
- * @param a Beginning of range
- * @param b End of range
- * @param num Number of times to split range. Pass 1 if you want to split
- * the range in two; i.e. one split.
- * @return Array of dividing values
- */
- interface Comparerruns with a sequence of run lengths of the form
- * Y_n x_{n,1} x_{n,2} ... x_{n,l_n}
- * Y_{n-1} x_{n-1,1} x_{n-1,2} ... x_{n-1,l_{n-1}}
- * ...
- * Y_1 x_{1,1} x_{1,2} ... x_{1,l_1}
- * The Y_i's are chosen to satisfy the invariant throughout execution,
- * but the x_{i,j}'s are merged (by TimSort.mergeCollapse)
- * into an X_i that violates the invariant.
- *
- * @param length The sum of all run lengths that will be added to runs.
- */
- private static Listruns such that:
- * 1. X = x_1 + ... + x_n
- * 2. x_j >= minRun for all j
- * 3. x_1 + ... + x_{j-2} < x_j < x_1 + ... + x_{j-1} for all j
- * These conditions guarantee that TimSort merges all x_j's one by one
- * (resulting in X) using only merges on the second-to-last element.
- *
- * @param X The sum of the sequence that should be added to runs.
- */
- private static void generateJDKWrongElem(List