Multishard with Remote Utxo Index and Random

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 3.5)
 project(codablecash)
 
 
@@ -7,11 +7,11 @@ add_subdirectory(src_ext)
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
 	message(STATUS "Debug Build")
 
-	set(CMAKE_CXX_FLAGS "-std=c++0x -D__cplusplus=201103L -D__DEBUG__ -D__USE_LARGEFILE64 -fmessage-length=0 -g -O0 -ftest-coverage -fprofile-arcs")
-	#set(CMAKE_CXX_FLAGS "-std=c++0x -D__cplusplus=201103L -D__DEBUG__ -fmessage-length=0 -O3 -march=native -MMD -MP -MF")
+	set(CMAKE_CXX_FLAGS "-g -O0 -std=c++11 -D__cplusplus=201103L -D__DEBUG__ -D__USE_LARGEFILE64 -fmessage-length=0 -coverage")
+	#set(CMAKE_CXX_FLAGS "-std=c++0x -D__cplusplus=201103L -D__DEBUG__ -fmessage-length=0 -O3 -march=native -MMD -MP -MF -ftest-coverage")
 
-	#set(CMAKE_C_FLAGS "-g -O0 -coverage -fprofile-arcs -ftest-coverage")
-	#set(CMAKE_EXE_LINKER_FLAGS="-fprofile-arcs -ftest-coverage")
+	#set(CMAKE_C_FLAGS "-g -O0 -coverage -coverage")
+	#set(CMAKE_EXE_LINKER_FLAGS="-ftest-coverage")
 
 	set(CMAKE_C_FLAGS "-g -O0 -coverage")
 	set(CMAKE_EXE_LINKER_FLAGS=" -coverage")
@@ -23,7 +23,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug")
 elseif(CMAKE_BUILD_TYPE STREQUAL "ReleaseTest")
 	set(CMAKE_CXX_FLAGS "-std=c++0x -D__cplusplus=201103L -D__DEBUG__ -D__USE_LARGEFILE64 -fmessage-length=0 -g -march=native -MMD -MP -MF")
 else()
-	set(CMAKE_CXX_FLAGS "-std=c++0x -D__cplusplus=201103L -DNDEBUG -D__USE_LARGEFILE64 -Wall -fmessage-length=0 -O3 -march=native -MMD -MP -MF")
+	set(CMAKE_CXX_FLAGS "-std=c++0x -D__cplusplus=201103L -DNDEBUG -D__USE_LARGEFILE64 -Wall -fmessage-length=0 -O2 -march=native -MMD -MP -MF")
 endif()
 
 
@@ -136,6 +136,7 @@ include_directories(${PROJECT_SOURCE_DIR}/src_gen/)
 include_directories(${PROJECT_SOURCE_DIR}/src_ext/)
 include_directories(${PROJECT_SOURCE_DIR}/src_smartcontract/)
 include_directories(${PROJECT_SOURCE_DIR}/src_smartcontract_db/)
+include_directories(${PROJECT_SOURCE_DIR}/src_smartcontract_modular/)
 include_directories(${PROJECT_SOURCE_DIR}/src_smartcontract_vm/)
 include_directories(${PROJECT_SOURCE_DIR}/src_test/)
 
@@ -146,10 +147,11 @@ add_subdirectory(src_db)
 add_subdirectory(src_gen)
 add_subdirectory(src_smartcontract)
 add_subdirectory(src_smartcontract_db)
+add_subdirectory(src_smartcontract_modular)
 add_subdirectory(src_smartcontract_vm)
 
 
-add_library(codablecashlib STATIC ${top_src} ${top_src_blockchain} ${top_src_blockchain_p2p} ${top_src_db} ${top_src_gen} ${top_src_smartcontract} ${top_src_smartcontract_db} ${top_src_smartcontract_vm})
+add_library(codablecashlib STATIC ${top_src} ${top_src_blockchain} ${top_src_blockchain_p2p} ${top_src_db} ${top_src_gen} ${top_src_smartcontract} ${top_src_smartcontract_db} ${top_src_smartcontract_modular} ${top_src_smartcontract_vm})
 target_link_libraries(codablecashlib extlib pthread gmp)
 
 add_executable(codablecash ${srcmain})
@@ -191,22 +193,22 @@ add_custom_target(cppcheck
 
 add_custom_target(valgrind
 	COMMAND valgrind --leak-check=full
-		--xml=yes --xml-file=testall.%p.valgrind.xml --main-stacksize=83886080
+		--xml=yes --xml-file=testall.%p.valgrind.xml --main-stacksize=83886080 --max-threads=1000 --fair-sched=yes
 		./testall
 
 	WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 	)
 
 add_custom_target(valgrindp
 	COMMAND valgrind --leak-check=full
-		--xml=yes --xml-file=testall.%p.valgrind.xml --main-stacksize=83886080
-		./testall -g TestVoteAndFinalizeTicketGroup
+		--xml=yes --xml-file=testall.%p.valgrind.xml --main-stacksize=83886080 --max-threads=1000 --fair-sched=yes
+		./testall -g TestMultiShardGroup
 
 	WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 	)
 
 add_custom_target(helgrindp
-	COMMAND valgrind --tool=helgrind --xml=yes --xml-file=testall.%p.helgrind.xml ./testall -g TestPoWManagerGroup
+	COMMAND valgrind --tool=helgrind --xml=yes --xml-file=testall.%p.helgrind.xml ./testall -g TestSyncHeaderOnlyGroup
 
 	WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 	)

src/crypto/Schnorr.cpp

@@ -47,7 +47,7 @@ SchnorrKeyPair* Schnorr::generateKey(BigInteger seed) {
 }
 
 SchnorrKeyPair* Schnorr::generateKey(){
-	BigInteger s = BigInteger::ramdom();
+	BigInteger s = BigInteger::random();
 
 	return generateKey(s);
 }
@@ -59,7 +59,7 @@ SchnorrSignature* codablecash::Schnorr::sign(const BigInteger& s, const BigInteg
 }
 
 SchnorrSignature* Schnorr::sign(const BigInteger& s, const BigInteger& p, const uint8_t* data, size_t size){
-	BigInteger r = BigInteger::ramdom().modSelf(cnsts.Q_1);
+	BigInteger r = BigInteger::random().modSelf(cnsts.Q_1);
 	BigInteger powG = cnsts.G.modPow(r, cnsts.Q);
 
 	BigInteger* e = nullptr;

src/ecda/ScPrivateKey.cpp

@@ -23,7 +23,7 @@ ScPrivateKey::ScPrivateKey(const BigInteger* seed, uint64_t solt) : keyvalue((in
 }
 
 ScPrivateKey::ScPrivateKey() : keyvalue((int64_t)0) {
-	this->keyvalue = BigInteger::ramdom().mod(ScPrivateKey::p);
+	this->keyvalue = BigInteger::random().mod(ScPrivateKey::p);
 }
 
 ScPrivateKey::~ScPrivateKey() {

src/ecda/ScSignature.cpp

@@ -29,7 +29,7 @@ ScSignature::~ScSignature() {
 void ScSignature::sign(ByteBuffer* data, const BigInteger s) {
 	Secp256k1Point G;
 
-	BigInteger r = BigInteger::ramdom().mod(Secp256k1Point::p);
+	BigInteger r = BigInteger::random().mod(Secp256k1Point::p);
 
 	Secp256k1Point rG = G.multiple(r);
 

src/musig/SimpleMuSigSigner.cpp

@@ -27,7 +27,7 @@ Secp256k1Point SimpleMuSigSigner::getxG() {
 }
 
 Secp256k1Point SimpleMuSigSigner::getrG() {
-	this->r = BigInteger::ramdom(BigInteger(0L), Secp256k1Point::n);
+	this->r = BigInteger::random(BigInteger(0L), Secp256k1Point::n);
 	Secp256k1Point G;
 
 	return G.multiple(this->r);

src_blockchain/CMakeLists.txt

@@ -30,6 +30,7 @@ add_subdirectory(bc_status_cache)
 add_subdirectory(bc_status_cache_context)
 add_subdirectory(bc_status_cache_context_finalizer)
 add_subdirectory(bc_status_cache_data)
+add_subdirectory(bc_status_cache_extend_shard)
 add_subdirectory(bc_status_cache_lockin)
 add_subdirectory(bc_status_cache_vote)
 add_subdirectory(bc_trx)

src_blockchain/bc/CodablecashNodeInstance.cpp

@@ -183,7 +183,7 @@ void CodablecashNodeInstance::startNetwork(const UnicodeString *host, int port)
 	this->p2pServer = new P2pServer(this->logger, this);
 	this->p2pManager = new BlochchainP2pManager();
 
-	this->p2pManager->init(this->blockchain->getNumZones());
+	this->p2pManager->init(this->statusCache->getNumZones());
 
 	this->p2pServer->addConnectionListener(this->p2pManager);
 
@@ -233,7 +233,7 @@ void CodablecashNodeInstance::shutdownNetwork() {
 }
 
 void CodablecashNodeInstance::startBlockGenerator(const MiningConfig *config) {
-	uint16_t zone = this->blockchain->getZoneSelf();
+	uint16_t zone = this->statusCache->getZoneSelf();
 
 	this->powManager = new PoWManager(this->logger, config);
 	this->blockGenerator = this->allocator->newBlockGenerator(zone, this->param, this->memoryPool, this->ctrl, config, this->logger);
@@ -378,7 +378,7 @@ void CodablecashNodeInstance::loginNode(uint16_t zone, P2pHandshake *handshake,
 	// login
 	NodeIdentifierSource* source = this->p2pRequestProcessor->getNetworkKey();
 
-	uint16_t zoneSelf = this->blockchain->getZoneSelf();
+	uint16_t zoneSelf = this->statusCache->getZoneSelf();
 	LoginPubSubCommand cmd(zoneSelf, canonicalName);
 
 	{
@@ -414,11 +414,11 @@ NodeIdentifierSource* CodablecashNodeInstance::getNetworkKey() const noexcept  {
 }
 
 uint16_t CodablecashNodeInstance::getZoneSelf() const noexcept {
-	return this->blockchain->getZoneSelf();
+	return this->statusCache->getZoneSelf();
 }
 
 int CodablecashNodeInstance::getNumZones() const noexcept {
-	return this->blockchain->getNumZones();
+	return this->statusCache->getNumZones();
 }
 
 void CodablecashNodeInstance::maintainNetwork() {
@@ -495,7 +495,7 @@ void CodablecashNodeInstance::setNodeName(const UnicodeString *name) noexcept {
 }
 
 void CodablecashNodeInstance::validateZone(uint16_t zone) const {
-	uint16_t numZone = this->blockchain->getNumZones();
+	uint16_t numZone = this->statusCache->getNumZones();
 
 	ExceptionThrower<InvalidZoneException>::throwExceptionIfCondition(numZone <= zone, L"", __FILE__, __LINE__);
 }
@@ -515,4 +515,8 @@ int CodablecashNodeInstance::getListningPort() const noexcept {
 	return this->p2pServer->getListningPort();
 }
 
+void CodablecashNodeInstance::setShardExtendValidator(const AbstractShardExtentionValidator *validator) {
+	this->statusCache->setShardExtentionValidator(validator);
+}
+
 } /* namespace codablecash */

src_blockchain/bc/CodablecashNodeInstance.h

@@ -39,6 +39,8 @@ class P2pDnsManager;
 class P2pHandshake;
 class P2pNodeRecord;
 class NodeIdentifier;
+class AbstractShardExtentionValidator;
+
 
 class CodablecashNodeInstance : public IPubsubCommandExecutor {
 public:
@@ -132,6 +134,8 @@ class CodablecashNodeInstance : public IPubsubCommandExecutor {
 		return this->blockGenerator;
 	}
 
+	void setShardExtendValidator(const AbstractShardExtentionValidator *validator);
+
 private:
 	void __init(const File* baseDir, ISystemLogger* logger, const CodablecashSystemParam* config);
 	void __maintainNetwork(uint16_t zone);

src_blockchain/bc/CodablecashSystemParam.cpp

@@ -46,6 +46,9 @@ CodablecashSystemParam::CodablecashSystemParam(const CodablecashSystemParam &ins
 	this->consensusTrxAllowedDelayMillis = inst.consensusTrxAllowedDelayMillis;
 
 	this->consensusPosVoteLimitMillis = inst.consensusPosVoteLimitMillis;
+
+	this->remoteUtxoSaveHeightPeriod = inst.remoteUtxoSaveHeightPeriod;
+	this->remoteUtxoExpireHeight = inst.remoteUtxoExpireHeight;
 }
 
 CodablecashSystemParam::CodablecashSystemParam() {
@@ -85,6 +88,9 @@ CodablecashSystemParam::CodablecashSystemParam() {
 	this->consensusTrxAllowedDelayMillis = 3000;
 
 	this->consensusPosVoteLimitMillis = 20000;
+
+	this->remoteUtxoSaveHeightPeriod = 10000;
+	this->remoteUtxoExpireHeight = 5000;
 }
 
 CodablecashSystemParam::~CodablecashSystemParam() {

src_blockchain/bc/CodablecashSystemParam.h

@@ -12,6 +12,9 @@
 
 namespace codablecash {
 
+class BlockchainSoftwareVersion;
+
+
 class CodablecashSystemParam {
 public:
 	CodablecashSystemParam(const CodablecashSystemParam& inst);
@@ -112,6 +115,13 @@ class CodablecashSystemParam {
 		return this->consensusPosVoteLimitMillis;
 	}
 
+	uint64_t getRemoteUtxoSaveHeightPeriod(const BlockchainSoftwareVersion* version) const noexcept {
+		return this->remoteUtxoSaveHeightPeriod;
+	}
+	uint64_t getRemoteUtxoExpireHeight(const BlockchainSoftwareVersion* version) const noexcept {
+		return this->remoteUtxoExpireHeight;
+	}
+
 private:
 	// pow
 	uint16_t powHashrateBlocks;
@@ -160,6 +170,10 @@ class CodablecashSystemParam {
 	// Pos Vote Limit
 	uint32_t consensusPosVoteLimitMillis;
 
+	// Remote UtxoId to Store period by height
+	uint64_t remoteUtxoSaveHeightPeriod;
+	uint64_t remoteUtxoExpireHeight;
+
 };
 
 } /* namespace codablecash */

src_blockchain/bc_base/Abstract32BytesId.cpp

@@ -99,7 +99,7 @@ ByteBuffer* Abstract32BytesId::makeRandom16Bytes() {
 	int size = 0;
 	BigInteger p(L"0", 16);
 	do{
-		BigInteger seed = BigInteger::ramdom();
+		BigInteger seed = BigInteger::random();
 
 		BigInteger s = seed.mod(Abstract32BytesId::Q);
 		p = G.modPow(s, Abstract32BytesId::Q);

src_blockchain/bc_base/AbstractAddress.cpp

@@ -58,7 +58,7 @@ AddressDescriptor* AbstractAddress::toAddressDescriptor() const noexcept {
 
 	char zonech[4];
 	Mem::memset(zonech, 0, 4);
-	::sprintf(zonech, "%03d", this->zone);
+	::sprintf(zonech, "%03d", (uint8_t)this->zone);
 
 	ByteBuffer* body = getBodyPart(); __STP(body);
 	body->position(0);

src_blockchain/bc_base_trx_index/TransactionDataFactory.cpp

@@ -27,11 +27,8 @@ IBlockObject* TransactionDataFactory::makeDataFromBinary(ByteBuffer *in) {
 	return TransactionData::fromBinary(in);
 }
 
-void TransactionDataFactory::registerData(const AbstractBtreeKey *key,
-		const IBlockObject *data, DataNode *dataNode,
-		BtreeStorage *store) const {
-	uint64_t dataFpos = store->storeData(data);
-	dataNode->setDataFpos(dataFpos);
+void TransactionDataFactory::registerData(const AbstractBtreeKey *key, const IBlockObject *data, DataNode *dataNode, BtreeStorage *store) const {
+	AbstractBtreeDataFactory::registerData(key, data, dataNode, store);
 }
 
 bool TransactionDataFactory::beforeRemove(DataNode *dataNode,

src_blockchain/bc_base_trx_index/TransactionIdDataFactory.cpp

@@ -27,8 +27,7 @@ IBlockObject* TransactionIdDataFactory::makeDataFromBinary(ByteBuffer *in) {
 
 void TransactionIdDataFactory::registerData(const AbstractBtreeKey *key,
 		const IBlockObject *data, DataNode *dataNode, BtreeStorage *store) const {
-	uint64_t dataFpos = store->storeData(data);
-	dataNode->setDataFpos(dataFpos);
+	AbstractBtreeDataFactory::registerData(key, data, dataNode, store);
 }
 
 bool TransactionIdDataFactory::beforeRemove(DataNode *dataNode,

src_blockchain/bc_base_utxo_index/AddressDescriptorDataFactory.cpp

@@ -27,10 +27,8 @@ IBlockObject* AddressDescriptorDataFactory::makeDataFromBinary(ByteBuffer *in) {
 }
 
 void AddressDescriptorDataFactory::registerData(const AbstractBtreeKey *key,
-		const IBlockObject *data, DataNode *dataNode,
-		BtreeStorage *store) const {
-	uint64_t dataFpos = store->storeData(data);
-	dataNode->setDataFpos(dataFpos);
+		const IBlockObject *data, DataNode *dataNode, BtreeStorage *store) const {
+	AbstractBtreeDataFactory::registerData(key, data, dataNode, store);
 }
 
 bool AddressDescriptorDataFactory::beforeRemove(DataNode *dataNode,

src_blockchain/bc_base_utxo_index/UtxoDataFactory.cpp

@@ -28,8 +28,7 @@ IBlockObject* UtxoDataFactory::makeDataFromBinary(ByteBuffer *in) {
 
 void UtxoDataFactory::registerData(const AbstractBtreeKey *key,
 		const IBlockObject *data, DataNode *dataNode, BtreeStorage *store) const {
-	uint64_t dataFpos = store->storeData(data);
-	dataNode->setDataFpos(dataFpos);
+	AbstractBtreeDataFactory::registerData(key, data, dataNode, store);
 }
 
 bool UtxoDataFactory::beforeRemove(DataNode *dataNode, BtreeStorage *store, const AbstractBtreeKey *key) const {

src_blockchain/bc_block/Block.cpp

@@ -169,4 +169,8 @@ bool Block::isScheduledBlock() const noexcept {
 	return this->header->isScheduledBlock();
 }
 
+void Block::addHeaderCommand(const AbstractBlockHeaderCommand *cmd) {
+	this->header->addHeaderCommand(cmd);
+}
+
 } /* namespace codablecash */

src_blockchain/bc_block/Block.h

@@ -29,6 +29,8 @@ class BlockHeaderId;
 class PoWNonce;
 class VoteBlockTransaction;
 class ReservedVotes;
+class AbstractBlockHeaderCommand;
+
 
 class Block {
 public:
@@ -70,6 +72,8 @@ class Block {
 
 	bool isScheduledBlock() const noexcept;
 
+	void addHeaderCommand(const AbstractBlockHeaderCommand *cmd);
+
 #ifdef __DEBUG__
 	bool checkMerkleRoot() const noexcept;
 #endif

src_blockchain/bc_block/BlockHeader.cpp

@@ -30,6 +30,7 @@
 
 #include "bc_base/BinaryUtils.h"
 
+#include "bc_block_header_command/BlockHeaderCommandId.h"
 
 namespace codablecash {
 
@@ -303,4 +304,21 @@ bool BlockHeader::hasHeaderCommnads() const noexcept {
 	return !this->commnads->isEmpty();
 }
 
+const AbstractBlockHeaderCommand* BlockHeader::getHeaderCommand(const BlockHeaderCommandId *commandId) const noexcept {
+	const AbstractBlockHeaderCommand* ret = nullptr;
+
+	int maxLoop = this->commnads->size();
+	for(int i = 0; i != maxLoop; ++i){
+		AbstractBlockHeaderCommand* command = this->commnads->get(i);
+		const BlockHeaderCommandId* id = command->getCommandId();
+
+		if(commandId->equals(id)){
+			ret = command;
+			break;
+		}
+	}
+
+	return ret;
+}
+
 } /* namespace codablecash */