Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ set(CMAKE_CXX_EXTENSIONS OFF)
# Generate compile_commands.json
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

add_compile_options(-Wunused -Wunused-function)

# ------------------------------------------------------------------------------
# GoogleTest (submodule)
# ------------------------------------------------------------------------------
Expand Down
1 change: 1 addition & 0 deletions example/common/tiny_shakespeare_dataset.cc
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ TinyShakespeareFile ReadTinyShakespeareFile(const std::string &path, size_t sequ
const auto header = ReadSeveralBytesFromIfstream(1024, &ifs);
const int magic = BytesToType<int32_t>(header, 0);
const int version = BytesToType<int32_t>(header, 4);
(void)version; // Read but unused; reserved in binary format specification
const int num_tokens = BytesToType<int32_t>(header, 8);
text_file.type = kTypeMap.at(magic);

Expand Down
3 changes: 0 additions & 3 deletions example/gpt2/checkpoint_loader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,8 @@ std::shared_ptr<nn::TransformerModel> LoadFromLLMC(const std::string &filepath)
// calculate xx_size_per_partition
const int64_t vpp = model_vocab_size / tp_size;
const int64_t v_start = static_cast<int64_t>(tp_rank) * vpp;
const int64_t v_end = v_start + vpp;

const int64_t qkv_out = 3 * n_embd;
const int64_t qkv_pp = qkv_out / tp_size;
const int64_t qkv_start = static_cast<int64_t>(tp_rank) * qkv_pp;

const int64_t fc_out = 4 * n_embd;
const int64_t fc_pp = fc_out / tp_size;
Expand Down
1 change: 0 additions & 1 deletion example/llama3/checkpoint_loader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,6 @@ std::shared_ptr<nn::TransformerModel> LoadFromLLMC(const std::string &filepath)
const int64_t q_local_rows = static_cast<int64_t>(n_embd) / tp_size; // = (n_head/world)*head_dim
const int64_t kv_head_local = static_cast<int64_t>(n_kv_head) / tp_size;
const int64_t kv_local_rows = kv_head_local * head_dim; // for K or V (each)
const int64_t attn_local_rows = q_local_rows + 2 * kv_local_rows;

// RowParallel (proj)
const int64_t in_pp = static_cast<int64_t>(n_embd) / tp_size;
Expand Down
2 changes: 1 addition & 1 deletion infini_train/src/nn/parallel/ddp/param_and_grad_buffer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ void ParamAndGradBucketGroup::RegisterGradReady(const std::shared_ptr<Tensor> &p
return;
}

const bool inserted = params_with_grad_.insert(parameter.get()).second;
params_with_grad_.insert(parameter.get());
// TODO(zbl): check this if sync is only done in last mircobatch
// if (!inserted) {
// LOG(FATAL) << "ParamAndGradBucketGroup: RegisterGradReady() was called twice for the same parameter in a
Expand Down
5 changes: 0 additions & 5 deletions infini_train/src/nn/parallel/global.cc
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,6 @@ int GetEnvAsInt(const std::string &name, int default_value) {
return value ? std::atoi(value) : default_value;
}

std::string GetEnvAsStr(const std::string &name, const std::string &default_value) {
const char *value = std::getenv(name.c_str());
return value ? std::string(value) : default_value;
}

} // namespace

namespace infini_train::nn::parallel::global {
Expand Down
Loading