Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions .github/workflows/cluster-test-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,27 @@ jobs:

- name: Run simple cluster test
run: |
mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test -am -P simple-cluster-test
timeout 45m mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test \
-am -P simple-cluster-test

- name: Run multi cluster test
run: |
mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test -am -P multi-cluster-test
timeout 45m mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test \
-am -P multi-cluster-test

- name: Show cluster diagnostics on failure
if: failure()
run: |
echo "[ci] java processes:"
ps -ef | grep -E "HugeGraph|hg-|java" | grep -v grep || true
echo "[ci] listening tcp ports:"
(ss -ltnp || netstat -ltnp || true) 2>&1
find hugegraph-cluster-test -path '*/logs/*' -type f | sort | while read -r log; do
echo "--- tail -n 200 $log ---"
tail -n 200 "$log" || true
done
find . -path '*/surefire-reports/*' -type f \
\( -name '*.txt' -o -name '*.xml' \) | sort | while read -r report; do
echo "--- tail -n 120 $report ---"
tail -n 120 "$report" || true
done
4 changes: 2 additions & 2 deletions .github/workflows/commons-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ on:
push:
branches:
- master
- /^release-.*$/
- /^test-.*$/
- 'release-*'
- 'test-*'
pull_request:

jobs:
Expand Down
72 changes: 68 additions & 4 deletions .github/workflows/pd-store-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,23 @@ jobs:
run: |
mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-rest-test

- name: Show PD diagnostics on failure
if: failure()
run: |
VERSION=$(grep -E '^VersionInBash=' \
hugegraph-commons/hugegraph-common/src/main/resources/version.properties | cut -d'=' -f2-)
[[ "$VERSION" =~ ^[0-9A-Za-z._-]+$ ]] || {
echo "Invalid VersionInBash: $VERSION"
exit 1
}
PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION
bash $TRAVIS_DIR/ci-service-utils.sh dump "$PD_DIR" HugeGraphPD || true
find . -path '*/surefire-reports/*' -type f \
\( -name '*.txt' -o -name '*.xml' \) | sort | while read -r report; do
echo "--- tail -n 120 $report ---"
tail -n 120 "$report" || true
done

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3.0.0
with:
Expand Down Expand Up @@ -246,6 +263,25 @@ jobs:
run: |
mvn test -pl hugegraph-store/hg-store-test -am -P store-raftcore-test

- name: Show Store diagnostics on failure
if: failure()
run: |
VERSION=$(grep -E '^VersionInBash=' \
hugegraph-commons/hugegraph-common/src/main/resources/version.properties | cut -d'=' -f2-)
[[ "$VERSION" =~ ^[0-9A-Za-z._-]+$ ]] || {
echo "Invalid VersionInBash: $VERSION"
exit 1
}
PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION
STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION
bash $TRAVIS_DIR/ci-service-utils.sh dump "$PD_DIR" HugeGraphPD || true
bash $TRAVIS_DIR/ci-service-utils.sh dump "$STORE_DIR" HugeGraphStore || true
find . -path '*/surefire-reports/*' -type f \
\( -name '*.txt' -o -name '*.xml' \) | sort | while read -r report; do
echo "--- tail -n 120 $report ---"
tail -n 120 "$report" || true
done

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3.0.0
with:
Expand Down Expand Up @@ -312,10 +348,38 @@ jobs:
run: |
$TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR

- name: Run TinkerPop test
if: ${{ env.RELEASE_BRANCH == 'true' }}
run: |
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND tinkerpop
# HStore remains covered by unit/core/API here; TinkerPop compliance is
# exercised in Server CI on memory and rocksdb.
- name: Run TinkerPop structure test
if: ${{ env.RELEASE_BRANCH == 'true' && env.BACKEND != 'hstore' }}
timeout-minutes: 60
run: |
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND structure

- name: Run TinkerPop process test
if: ${{ env.RELEASE_BRANCH == 'true' && env.BACKEND != 'hstore' }}
timeout-minutes: 60
run: |
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND process

- name: Show HStore diagnostics on failure
if: failure()
run: |
VERSION=$(grep -E '^VersionInBash=' \
hugegraph-commons/hugegraph-common/src/main/resources/version.properties | cut -d'=' -f2-)
[[ "$VERSION" =~ ^[0-9A-Za-z._-]+$ ]] || {
echo "Invalid VersionInBash: $VERSION"
exit 1
}
PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION
STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION
bash $TRAVIS_DIR/ci-service-utils.sh dump "$PD_DIR" HugeGraphPD || true
bash $TRAVIS_DIR/ci-service-utils.sh dump "$STORE_DIR" HugeGraphStore || true
find . -path '*/surefire-reports/*' -type f \
\( -name '*.txt' -o -name '*.xml' \) | sort | while read -r report; do
echo "--- tail -n 120 $report ---"
tail -n 120 "$report" || true
done

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3.0.0
Expand Down
29 changes: 23 additions & 6 deletions .github/workflows/server-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,14 @@ jobs:
HEAD_BRANCH_NAME: ${{ github.head_ref }}
BASE_BRANCH_NAME: ${{ github.base_ref }}
TARGET_BRANCH_NAME: ${{ github.base_ref != '' && github.base_ref || github.ref_name }}
RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') }}
RAFT_MODE: ${{ startsWith(github.head_ref, 'test') || startsWith(github.head_ref, 'raft') }}
RELEASE_BRANCH: >-
${{
startsWith(github.ref_name, 'release-') ||
startsWith(github.ref_name, 'test-') ||
startsWith(github.head_ref, 'release-') ||
startsWith(github.head_ref, 'test-')
}}
RAFT_MODE: ${{ startsWith(github.ref_name, 'raft-') || startsWith(github.head_ref, 'raft-') }}

strategy:
fail-fast: false
Expand Down Expand Up @@ -115,14 +121,25 @@ jobs:
# TODO: disable raft test in normal PR due to the always timeout problem
- name: Run raft test
if: ${{ env.RAFT_MODE == 'true' && env.BACKEND == 'rocksdb' }}
timeout-minutes: 45
run: |
$TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR

- name: Run TinkerPop test
if: ${{ env.RELEASE_BRANCH == 'true' }}
# TinkerPop compliance is covered by memory and rocksdb in CI.
# HBase still runs compile/unit/core/API because its full suite exceeds the CI budget.
- name: Run TinkerPop structure test
if: ${{ env.RELEASE_BRANCH == 'true' && env.BACKEND != 'hbase' }}
timeout-minutes: 60
run: |
echo "[WARNING] Enter Tinkerpop Test, current 'github.ref_name' is ${{ github.ref_name }}"
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND tinkerpop
echo "[WARNING] Enter Tinkerpop Structure Test, current 'github.ref_name' is ${{ github.ref_name }}"
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND structure

- name: Run TinkerPop process test
if: ${{ env.RELEASE_BRANCH == 'true' && env.BACKEND != 'hbase' }}
timeout-minutes: 60
run: |
echo "[WARNING] Enter Tinkerpop Process Test, current 'github.ref_name' is ${{ github.ref_name }}"
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND process

- name: Upload coverage to Codecov
# TODO: update to v5 later
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ HugeGraph supports both **standalone** and **distributed** deployments:
│ HugeGraph Server (:8080) │
│ ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │
│ │ REST API │ │ Gremlin │ │ Cypher Engine │ │
│ │(Jersey 3)│ │ (TP 3.5) │ │ (OpenCypher) │ │
│ │(Jersey 3)│ │ (TP 3.7) │ │ (OpenCypher) │ │
│ └────┬─────┘ └────┬─────┘ └────────┬─────────┘ │
│ └─────────────┼─────────────────┘ │
│ ┌────────▼────────┐ │
Expand Down Expand Up @@ -128,7 +128,7 @@ flowchart TB

subgraph Server["HugeGraph Server :8080"]
API[REST API<br/>Jersey 3]
GS[Gremlin Server<br/>TinkerPop 3.5]
GS[Gremlin Server<br/>TinkerPop 3.7]
CS[Cypher Engine<br/>OpenCypher]
CORE[Graph Engine<br/>hugegraph-core]

Expand Down Expand Up @@ -294,7 +294,7 @@ curl http://localhost:8080/versions
# "versions": {
# "version": "v1",
# "core": "1.7.0",
# "gremlin": "3.5.1",
# "gremlin": "3.7.6",
# "api": "1.7.0"
# }
# }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

import org.apache.hugegraph.ct.base.HGTestLogger;
import org.apache.hugegraph.ct.config.ClusterConfig;
import org.apache.hugegraph.ct.config.GraphConfig;
import org.apache.hugegraph.ct.config.PDConfig;
import org.apache.hugegraph.ct.config.ServerConfig;
import org.apache.hugegraph.ct.config.StoreConfig;
import org.apache.hugegraph.ct.node.AbstractNodeWrapper;
import org.apache.hugegraph.ct.node.PDNodeWrapper;
import org.apache.hugegraph.ct.node.ServerNodeWrapper;
import org.apache.hugegraph.ct.node.StoreNodeWrapper;
Expand All @@ -40,6 +42,8 @@
public abstract class AbstractEnv implements BaseEnv {

private static final Logger LOG = HGTestLogger.ENV_LOG;
private static final int NODE_START_TIMEOUT_SECONDS = 120;
private static final int NODE_START_POLL_MILLIS = 1000;

protected ClusterConfig clusterConfig;
protected List<PDNodeWrapper> pdNodeWrappers;
Expand Down Expand Up @@ -88,34 +92,54 @@ protected void init(int pdCnt, int storeCnt, int serverCnt) {

public void startCluster() {
for (PDNodeWrapper pdNodeWrapper : pdNodeWrappers) {
pdNodeWrapper.start();
while (!pdNodeWrapper.isStarted()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
startNode(pdNodeWrapper);
}
for (StoreNodeWrapper storeNodeWrapper : storeNodeWrappers) {
storeNodeWrapper.start();
while (!storeNodeWrapper.isStarted()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
startNode(storeNodeWrapper);
}
for (ServerNodeWrapper serverNodeWrapper : serverNodeWrappers) {
serverNodeWrapper.start();
while (!serverNodeWrapper.isStarted()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
startNode(serverNodeWrapper);
}
}

private void startNode(AbstractNodeWrapper nodeWrapper) {
System.out.printf("[cluster-test] starting %s in %s%n",
nodeWrapper.getID(), nodeWrapper.getNodePath());
nodeWrapper.start();
waitUntilStarted(nodeWrapper);
}

private static void waitUntilStarted(AbstractNodeWrapper nodeWrapper) {
long deadline = System.nanoTime() +
TimeUnit.SECONDS.toNanos(NODE_START_TIMEOUT_SECONDS);
while (System.nanoTime() < deadline) {
if (nodeWrapper.isStarted()) {
System.out.printf("[cluster-test] %s started%n",
nodeWrapper.getID());
return;
}
if (!nodeWrapper.isAlive()) {
nodeWrapper.dumpLog();
throw new AssertionError(String.format(
"%s failed to start, process status: %s",
nodeWrapper.getID(), nodeWrapper.processStatus()));
}
sleepBeforeRetry();
}

nodeWrapper.dumpLog();
throw new AssertionError(String.format(
"%s did not start within %s seconds, process status: %s",
nodeWrapper.getID(), NODE_START_TIMEOUT_SECONDS,
nodeWrapper.processStatus()));
}

private static void sleepBeforeRetry() {
try {
TimeUnit.MILLISECONDS.sleep(NODE_START_POLL_MILLIS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new AssertionError("Interrupted while waiting node start", e);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,36 @@ public void stop() {
}

public boolean isAlive() {
return this.instance.isAlive();
return this.instance != null && this.instance.isAlive();
}

public String processStatus() {
if (this.instance == null) {
return "not started";
}
if (this.instance.isAlive()) {
return "alive";
}
return "exited with code " + this.instance.exitValue();
}

public void dumpLog() {
Path logPath = Paths.get(getLogPath());
System.out.println("===== " + getID() + " log: " + logPath + " =====");
if (!Files.exists(logPath)) {
System.out.println("Log file does not exist");
return;
}

try {
List<String> lines = Files.readAllLines(logPath, StandardCharsets.UTF_8);
int start = Math.max(0, lines.size() - 200);
for (int i = start; i < lines.size(); i++) {
System.out.println(lines.get(i));
}
} catch (IOException e) {
System.out.println("Failed to read log file: " + e.getMessage());
}
}

protected ProcessBuilder runCmd(List<String> startCmd, File stdoutFile) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;

import org.junit.Test;

Expand Down Expand Up @@ -81,9 +82,11 @@ public void testNow() {
@Test
public void testParseCornerDateValue() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
final Date expected = DateUtil.parse("0", "yyyy");
int threadCount = 10;
List<Thread> threads = new ArrayList<>(threadCount);
AtomicInteger errorCount = new AtomicInteger(0);
AtomicReference<Throwable> firstError = new AtomicReference<>();
for (int t = 0; t < threadCount; t++) {
Thread thread = new Thread(() -> {
try {
Expand All @@ -92,9 +95,9 @@ public void testParseCornerDateValue() throws InterruptedException {
throw new RuntimeException(e);
}
try {
Assert.assertEquals(new Date(-62167248343000L),
DateUtil.parse("0", "yyyy"));
} catch (Exception e) {
Assert.assertEquals(expected, DateUtil.parse("0", "yyyy"));
} catch (Throwable e) {
firstError.compareAndSet(null, e);
errorCount.incrementAndGet();
}
});
Expand All @@ -109,6 +112,15 @@ public void testParseCornerDateValue() throws InterruptedException {
thread.join();
}

Throwable error = firstError.get();
if (error != null) {
AssertionError assertion = new AssertionError(String.format(
"Expected concurrent parses to match " +
"baseline result, but got %s failures",
errorCount.get()));
assertion.initCause(error);
throw assertion;
}
Assert.assertEquals(0, errorCount.get());
}

Expand Down
Loading