diff --git a/packages/google-cloud-bigtable/CONTRIBUTING.rst b/packages/google-cloud-bigtable/CONTRIBUTING.rst index 73e933e3865d..7797cec7dd5d 100644 --- a/packages/google-cloud-bigtable/CONTRIBUTING.rst +++ b/packages/google-cloud-bigtable/CONTRIBUTING.rst @@ -185,7 +185,7 @@ Code samples and snippets live in the `samples/` catalogue. Feel free to provide more examples, but make sure to write tests for those examples. Each folder containing example code requires its own `noxfile.py` script which automates testing. If you decide to create a new folder, you can -base it on the `samples/data_client_async/snippets` folder (providing `noxfile.py` and +base it on the `samples/data_client/snippets` folder (providing `noxfile.py` and the requirements files). The tests will run against a real Google Cloud Project, so you should @@ -194,11 +194,11 @@ configure them just like the System Tests. - To run sample tests, you can execute:: # Run all tests in a folder - $ cd samples/data_client_async/snippets + $ cd samples/data_client/snippets $ nox -s py-3.14 # Run a single sample test - $ cd samples/data_client_async/snippets + $ cd samples/data_client/snippets $ nox -s py-3.14 -- -k ******************************************** diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py new file mode 100644 index 000000000000..2ccff906b908 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to run instance admin operations using BigtableInstanceAdminClient. + +Prerequisites: +- Create a Cloud Bigtable project. +- Set your Google Application Default Credentials. +""" + +import argparse + +from google.api_core.exceptions import NotFound +from google.cloud import bigtable_admin_v2 + + +def run_instance_operations(project_id, instance_id, cluster_id): + client = bigtable_admin_v2.BigtableInstanceAdminClient() + project_path = client.common_project_path(project_id) + instance_path = client.instance_path(project_id, instance_id) + + # [START bigtable_check_instance_exists] + try: + instance = client.get_instance(name=instance_path) + print(f"Instance {instance_id} already exists.") + except NotFound: + print(f"Instance {instance_id} does not exist.") + # [END bigtable_check_instance_exists] + + # [START bigtable_create_prod_instance] + cluster_path = client.cluster_path(project_id, instance_id, cluster_id) + cluster = bigtable_admin_v2.Cluster( + location=client.common_location_path(project_id, "us-central1-f"), + serve_nodes=1, + default_storage_type=bigtable_admin_v2.StorageType.SSD, + ) + instance_obj = bigtable_admin_v2.Instance( + display_name=instance_id, + labels={"prod-label": "prod-label"}, + type_=bigtable_admin_v2.Instance.Type.PRODUCTION, + ) + + try: + client.get_instance(name=instance_path) + except NotFound: + print("\nCreating an instance") + operation = client.create_instance( + parent=project_path, + instance_id=instance_id, + instance=instance_obj, + clusters={cluster_id: cluster}, + ) + operation.result(timeout=480) + print(f"\nCreated instance: {instance_id}") + # [END bigtable_create_prod_instance] + + # [START bigtable_list_instances] + print("\nListing instances:") + instances_response = client.list_instances(parent=project_path) + for instance in instances_response.instances: + print(instance.name.split("/")[-1]) + # [END bigtable_list_instances] + + # [START bigtable_get_instance] + inst = client.get_instance(name=instance_path) + print(f"\nName of instance: {inst.display_name}\nLabels: {dict(inst.labels)}") + # [END bigtable_get_instance] + + # [START bigtable_get_clusters] + print("\nListing clusters...") + clusters_response = client.list_clusters(parent=instance_path) + for cluster in clusters_response.clusters: + print(cluster.name.split("/")[-1]) + # [END bigtable_get_clusters] + + +def delete_instance(project_id, instance_id): + client = bigtable_admin_v2.BigtableInstanceAdminClient() + instance_path = client.instance_path(project_id, instance_id) + + # [START bigtable_delete_instance] + print("\nDeleting instance") + try: + client.delete_instance(name=instance_path) + print(f"Deleted instance: {instance_id}") + except NotFound: + print(f"Instance {instance_id} does not exist.") + # [END bigtable_delete_instance] + + +def add_cluster(project_id, instance_id, cluster_id): + client = bigtable_admin_v2.BigtableInstanceAdminClient() + instance_path = client.instance_path(project_id, instance_id) + cluster_path = client.cluster_path(project_id, instance_id, cluster_id) + + # [START bigtable_create_cluster] + print("\nListing clusters...") + clusters_response = client.list_clusters(parent=instance_path) + for cluster in clusters_response.clusters: + print(cluster.name.split("/")[-1]) + + new_cluster = bigtable_admin_v2.Cluster( + location=client.common_location_path(project_id, "us-central1-a"), + serve_nodes=1, + default_storage_type=bigtable_admin_v2.StorageType.SSD, + ) + try: + client.get_cluster(name=cluster_path) + print(f"\nCluster not created, as {cluster_id} already exists.") + except NotFound: + operation = client.create_cluster( + parent=instance_path, cluster_id=cluster_id, cluster=new_cluster + ) + operation.result(timeout=480) + print(f"\nCluster created: {cluster_id}") + # [END bigtable_create_cluster] + + +def delete_cluster(project_id, instance_id, cluster_id): + client = bigtable_admin_v2.BigtableInstanceAdminClient() + cluster_path = client.cluster_path(project_id, instance_id, cluster_id) + + # [START bigtable_delete_cluster] + print("\nDeleting cluster") + try: + client.delete_cluster(name=cluster_path) + print(f"Cluster deleted: {cluster_id}") + except NotFound: + print(f"\nCluster {cluster_id} does not exist.") + # [END bigtable_delete_cluster] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "command", + help="run, del-instance, add-cluster or del-cluster.", + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument("instance_id", help="ID of the Cloud Bigtable instance.") + parser.add_argument("cluster_id", help="ID of the Cloud Bigtable cluster.") + + args = parser.parse_args() + + if args.command.lower() == "run": + run_instance_operations(args.project_id, args.instance_id, args.cluster_id) + elif args.command.lower() == "del-instance": + delete_instance(args.project_id, args.instance_id) + elif args.command.lower() == "add-cluster": + add_cluster(args.project_id, args.instance_id, args.cluster_id) + elif args.command.lower() == "del-cluster": + delete_cluster(args.project_id, args.instance_id, args.cluster_id) diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/noxfile.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/noxfile.py new file mode 100644 index 000000000000..a2a659dcca04 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/noxfile.py @@ -0,0 +1,292 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +import sys +from pathlib import Path +from typing import Callable, Dict, Optional + +import nox + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8") + else: + session.install("flake8", "flake8-annotations") + + args = FLAKE8_COMMON_ARGS + [ + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# format = isort + black +# + + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( + "**/test_*.py", recursive=True + ) + test_list.extend(glob.glob("**/tests", recursive=True)) + + if len(test_list) == 0: + print("No tests found, skipping directory.") + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + elif "pytest-xdist" in packages: + concurrent_args.extend(["-n", "auto"]) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """Returns the root folder of the project.""" + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements-test.txt b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements-test.txt new file mode 100644 index 000000000000..e079f8a6038d --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements-test.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements.txt b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements.txt new file mode 100644 index 000000000000..67a1ea5b8d23 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable==2.35.0 +backoff==2.2.1 diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py new file mode 100644 index 000000000000..17da5fb84303 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py @@ -0,0 +1,47 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from .instanceadmin import ( + add_cluster, + delete_cluster, + delete_instance, + run_instance_operations, +) + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +INSTANCE_ID = f"instance-admin-{str(uuid.uuid4())[:16]}" +CLUSTER_ID = f"cluster-admin-{str(uuid.uuid4())[:16]}" +NEW_CLUSTER_ID = f"cluster-add-{str(uuid.uuid4())[:16]}" + + +def test_instance_operations(capsys): + run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + out, _ = capsys.readouterr() + assert f"Created instance: {INSTANCE_ID}" in out + assert "Listing instances:" in out + assert "Listing clusters..." in out + + add_cluster(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + out, _ = capsys.readouterr() + assert f"Cluster created: {NEW_CLUSTER_ID}" in out + + delete_cluster(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + out, _ = capsys.readouterr() + assert f"Cluster deleted: {NEW_CLUSTER_ID}" in out + + delete_instance(PROJECT, INSTANCE_ID) + out, _ = capsys.readouterr() + assert f"Deleted instance: {INSTANCE_ID}" in out diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/noxfile.py b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/noxfile.py new file mode 100644 index 000000000000..a2a659dcca04 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/noxfile.py @@ -0,0 +1,292 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +import sys +from pathlib import Path +from typing import Callable, Dict, Optional + +import nox + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8") + else: + session.install("flake8", "flake8-annotations") + + args = FLAKE8_COMMON_ARGS + [ + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# format = isort + black +# + + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( + "**/test_*.py", recursive=True + ) + test_list.extend(glob.glob("**/tests", recursive=True)) + + if len(test_list) == 0: + print("No tests found, skipping directory.") + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + elif "pytest-xdist" in packages: + concurrent_args.extend(["-n", "auto"]) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """Returns the root folder of the project.""" + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements-test.txt b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements-test.txt new file mode 100644 index 000000000000..f01fd134c400 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements-test.txt @@ -0,0 +1,2 @@ +pytest +google-cloud-testutils==1.7.0 diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements.txt b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements.txt new file mode 100644 index 000000000000..730d25dec63f --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable==2.35.0 diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin.py b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin.py new file mode 100644 index 000000000000..115b1e33246a --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run table admin operations using BigtableTableAdminClient. + +Prerequisites: +- Create a Cloud Bigtable instance. +- Set your Google Application Default Credentials. +""" + +import argparse +import datetime + +from google.cloud import bigtable_admin_v2 +from google.protobuf import duration_pb2 + +from ..utils import create_table_cm + + +def run_table_operations(project_id, instance_id, table_id): + client = bigtable_admin_v2.BigtableTableAdminClient() + instance_path = client.instance_path(project_id, instance_id) + table_path = client.table_path(project_id, instance_id, table_id) + + with create_table_cm(project_id, instance_id, table_id, verbose=False): + # [START bigtable_list_tables] + tables = client.list_tables(parent=instance_path) + print("Listing tables in current project...") + if tables.tables: + for tbl in tables.tables: + print(tbl.name.split("/")[-1]) + else: + print("No table exists in current project...") + # [END bigtable_list_tables] + + # [START bigtable_create_family_gc_max_age] + print("Creating column family cf1 with MaxAge GC Rule...") + max_age_rule = bigtable_admin_v2.GcRule( + max_age=duration_pb2.Duration(seconds=5 * 86400) + ) + cf1 = bigtable_admin_v2.ColumnFamily(gc_rule=max_age_rule) + mod1 = bigtable_admin_v2.ModifyColumnFamiliesRequest.Modification( + id="cf1", create=cf1 + ) + client.modify_column_families(name=table_path, modifications=[mod1]) + print("Created column family cf1 with MaxAge GC Rule.") + # [END bigtable_create_family_gc_max_age] + + # [START bigtable_create_family_gc_max_versions] + print("Creating column family cf2 with max versions GC rule...") + max_versions_rule = bigtable_admin_v2.GcRule(max_num_versions=2) + cf2 = bigtable_admin_v2.ColumnFamily(gc_rule=max_versions_rule) + mod2 = bigtable_admin_v2.ModifyColumnFamiliesRequest.Modification( + id="cf2", create=cf2 + ) + client.modify_column_families(name=table_path, modifications=[mod2]) + print("Created column family cf2 with Max Versions GC Rule.") + # [END bigtable_create_family_gc_max_versions] + + # [START bigtable_create_family_gc_union] + print("Creating column family cf3 with union GC rule...") + union_rule = bigtable_admin_v2.GcRule( + union=bigtable_admin_v2.GcRule.Union( + rules=[ + bigtable_admin_v2.GcRule( + max_age=duration_pb2.Duration(seconds=5 * 86400) + ), + bigtable_admin_v2.GcRule(max_num_versions=2), + ] + ) + ) + cf3 = bigtable_admin_v2.ColumnFamily(gc_rule=union_rule) + mod3 = bigtable_admin_v2.ModifyColumnFamiliesRequest.Modification( + id="cf3", create=cf3 + ) + client.modify_column_families(name=table_path, modifications=[mod3]) + print("Created column family cf3 with Union GC rule") + # [END bigtable_create_family_gc_union] + + # [START bigtable_create_family_gc_intersection] + print("Creating column family cf4 with Intersection GC rule...") + intersection_rule = bigtable_admin_v2.GcRule( + intersection=bigtable_admin_v2.GcRule.Intersection( + rules=[ + bigtable_admin_v2.GcRule( + max_age=duration_pb2.Duration(seconds=5 * 86400) + ), + bigtable_admin_v2.GcRule(max_num_versions=2), + ] + ) + ) + cf4 = bigtable_admin_v2.ColumnFamily(gc_rule=intersection_rule) + mod4 = bigtable_admin_v2.ModifyColumnFamiliesRequest.Modification( + id="cf4", create=cf4 + ) + client.modify_column_families(name=table_path, modifications=[mod4]) + print("Created column family cf4 with Intersection GC rule.") + # [END bigtable_create_family_gc_intersection] + + # [START bigtable_create_family_gc_nested] + print("Creating column family cf5 with a Nested GC rule...") + rule1 = bigtable_admin_v2.GcRule(max_num_versions=10) + rule2 = bigtable_admin_v2.GcRule( + intersection=bigtable_admin_v2.GcRule.Intersection( + rules=[ + bigtable_admin_v2.GcRule( + max_age=duration_pb2.Duration(seconds=30 * 86400) + ), + bigtable_admin_v2.GcRule(max_num_versions=2), + ] + ) + ) + nested_rule = bigtable_admin_v2.GcRule( + union=bigtable_admin_v2.GcRule.Union(rules=[rule1, rule2]) + ) + cf5 = bigtable_admin_v2.ColumnFamily(gc_rule=nested_rule) + mod5 = bigtable_admin_v2.ModifyColumnFamiliesRequest.Modification( + id="cf5", create=cf5 + ) + client.modify_column_families(name=table_path, modifications=[mod5]) + print("Created column family cf5 with a Nested GC rule.") + # [END bigtable_create_family_gc_nested] + + # [START bigtable_list_column_families] + print("Printing Column Family and GC Rule for all column families...") + table_obj = client.get_table(name=table_path) + for column_family_name, cf_obj in sorted(table_obj.column_families.items()): + print("Column Family:", column_family_name) + print("GC Rule:") + print(cf_obj.gc_rule) + # [END bigtable_list_column_families] + + # [START bigtable_update_gc_rule] + print("Updating column family cf1 GC rule...") + updated_cf1 = bigtable_admin_v2.ColumnFamily( + gc_rule=bigtable_admin_v2.GcRule(max_num_versions=1) + ) + mod_update = bigtable_admin_v2.ModifyColumnFamiliesRequest.Modification( + id="cf1", update=updated_cf1 + ) + client.modify_column_families(name=table_path, modifications=[mod_update]) + print("Updated column family cf1 GC rule\n") + # [END bigtable_update_gc_rule] + + # [START bigtable_delete_family] + print("Delete a column family cf2...") + mod_delete = bigtable_admin_v2.ModifyColumnFamiliesRequest.Modification( + id="cf2", drop=True + ) + client.modify_column_families(name=table_path, modifications=[mod_delete]) + print("Column family cf2 deleted successfully.") + # [END bigtable_delete_family] + + +def delete_table(project_id, instance_id, table_id): + client = bigtable_admin_v2.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + # [START bigtable_delete_table] + print("Deleting {} table.".format(table_id)) + client.delete_table(name=table_path) + print("Deleted {} table.".format(table_id)) + # [END bigtable_delete_table] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument("command", help="run or delete.") + parser.add_argument( + "--table", help="Cloud Bigtable Table name.", default="Hello-Bigtable" + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument( + "instance_id", help="ID of the Cloud Bigtable instance to connect to." + ) + + args = parser.parse_args() + + if args.command.lower() == "run": + run_table_operations(args.project_id, args.instance_id, args.table) + elif args.command.lower() == "delete": + delete_table(args.project_id, args.instance_id, args.table) diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin_test.py b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin_test.py new file mode 100644 index 000000000000..38700717377e --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin_test.py @@ -0,0 +1,41 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from .tableadmin import delete_table, run_table_operations + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"table-admin-{str(uuid.uuid4())[:16]}" + + +def test_run_table_operations(capsys): + run_table_operations(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + out, _ = capsys.readouterr() + assert "Listing tables in current project..." in out + assert "Created column family cf1 with MaxAge GC Rule." in out + assert "Created column family cf2 with Max Versions GC Rule." in out + assert "Created column family cf3 with Union GC rule" in out + assert "Created column family cf4 with Intersection GC rule." in out + assert "Created column family cf5 with a Nested GC rule." in out + assert "Printing Column Family and GC Rule for all column families..." in out + assert "Updated column family cf1 GC rule" in out + assert "Column family cf2 deleted successfully." in out + + +def test_delete_table(capsys): + delete_table(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + out, _ = capsys.readouterr() + assert f"Deleted {TABLE_ID} table." in out diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/main.py b/packages/google-cloud-bigtable/samples/data_client/hello/main.py new file mode 100644 index 000000000000..6cbef394e8b0 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/main.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations with the sync data APIs + +Prerequisites: + +- Create a Cloud Bigtable instance. + https://cloud.google.com/bigtable/docs/creating-instance +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials +""" + +import argparse + +# [START bigtable_hw_imports_data_client] +from google.cloud import bigtable +from google.cloud.bigtable.data import row_filters + +from ..utils import wait_for_table + +# [END bigtable_hw_imports_data_client] + +# use to ignore warnings +row_filters + + +def main(project_id, instance_id, table_id): + # [START bigtable_hw_connect_data_client] + client = bigtable.data.BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + # [END bigtable_hw_connect_data_client] + + # [START bigtable_hw_create_table_data_client] + from google.cloud.bigtable import column_family + + # the data client only supports the data API. Table creation is an admin operation + # use admin client to create the table + print("Creating the {} table.".format(table_id)) + admin_client = bigtable.Client(project=project_id, admin=True) + admin_instance = admin_client.instance(instance_id) + admin_table = admin_instance.table(table_id) + + print("Creating column family cf1 with Max Version GC rule...") + # Create a column family with GC policy : most recent N versions + # Define the GC policy to retain only the most recent 2 versions + max_versions_rule = column_family.MaxVersionsGCRule(2) + column_family_id = b"cf1" + column_families = {column_family_id: max_versions_rule} + if not admin_table.exists(): + admin_table.create(column_families=column_families) + else: + print("Table {} already exists.".format(table_id)) + # [END bigtable_hw_create_table_data_client] + + try: + # let table creation complete + wait_for_table(admin_table) + # [START bigtable_hw_write_rows_data_client] + print("Writing some greetings to the table.") + greetings = [b"Hello World!", b"Hello Cloud Bigtable!", b"Hello Python!"] + mutations = [] + column = b"greeting" + for i, value in enumerate(greetings): + row_key = f"greeting{i}".encode() + row_mutation = bigtable.data.RowMutationEntry( + row_key, bigtable.data.SetCell(column_family_id, column, value) + ) + mutations.append(row_mutation) + table.bulk_mutate_rows(mutations) + # [END bigtable_hw_write_rows_data_client] + + # [START bigtable_hw_create_filter_data_client] + # Create a filter to only retrieve the most recent version of the cell + # for each column across entire row. + row_filter = bigtable.data.row_filters.CellsColumnLimitFilter(1) + # [END bigtable_hw_create_filter_data_client] + + # [START bigtable_hw_get_with_filter_data_client] + # [START bigtable_hw_get_by_key_data_client] + print("Getting a single greeting by row key.") + key = "greeting0".encode() + + row = table.read_row(key, row_filter=row_filter) + cell = row.cells[0] + print(cell.value.decode("utf-8")) + # [END bigtable_hw_get_by_key_data_client] + # [END bigtable_hw_get_with_filter_data_client] + + # [START bigtable_hw_scan_with_filter_data_client] + # [START bigtable_hw_scan_all_data_client] + print("Scanning for all greetings:") + query = bigtable.data.ReadRowsQuery(row_filter=row_filter) + for row in table.read_rows(query): + cell = row.cells[0] + print(cell.value.decode("utf-8")) + # [END bigtable_hw_scan_all_data_client] + # [END bigtable_hw_scan_with_filter_data_client] + finally: + # [START bigtable_hw_delete_table_data_client] + print("Deleting the {} table.".format(table_id)) + admin_table.delete() + client.close() + # [END bigtable_hw_delete_table_data_client] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument( + "instance_id", help="ID of the Cloud Bigtable instance to connect to." + ) + parser.add_argument( + "--table", help="Table to create and destroy.", default="Hello-Bigtable" + ) + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/main_test.py b/packages/google-cloud-bigtable/samples/data_client/hello/main_test.py new file mode 100644 index 000000000000..77cab55a025d --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/main_test.py @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from .main import main + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"hello-world-test-{str(uuid.uuid4())[:16]}" + + +def test_main(capsys): + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/noxfile.py b/packages/google-cloud-bigtable/samples/data_client/hello/noxfile.py new file mode 100644 index 000000000000..a2a659dcca04 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/noxfile.py @@ -0,0 +1,292 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +import sys +from pathlib import Path +from typing import Callable, Dict, Optional + +import nox + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8") + else: + session.install("flake8", "flake8-annotations") + + args = FLAKE8_COMMON_ARGS + [ + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# format = isort + black +# + + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( + "**/test_*.py", recursive=True + ) + test_list.extend(glob.glob("**/tests", recursive=True)) + + if len(test_list) == 0: + print("No tests found, skipping directory.") + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + elif "pytest-xdist" in packages: + concurrent_args.extend(["-n", "auto"]) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """Returns the root folder of the project.""" + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/requirements-test.txt b/packages/google-cloud-bigtable/samples/data_client/hello/requirements-test.txt new file mode 100644 index 000000000000..e079f8a6038d --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/requirements-test.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/requirements.txt b/packages/google-cloud-bigtable/samples/data_client/hello/requirements.txt new file mode 100644 index 000000000000..5113ca7f17bb --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable==2.35.0 +google-cloud-core==2.5.0 diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/main.py b/packages/google-cloud-bigtable/samples/data_client/quickstart/main.py new file mode 100644 index 000000000000..dacbe6c7ae8c --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/main.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigtable_quickstart_data_client] +import argparse + +from google.cloud.bigtable.data import BigtableDataClient + + +def main(project_id="project-id", instance_id="instance-id", table_id="my-table"): + # Create a Cloud Bigtable client. + client = BigtableDataClient(project=project_id) + + # Open an existing table. + table = client.get_table(instance_id, table_id) + + row_key = "r1" + row = table.read_row(row_key) + + column_family_id = "cf1" + column_id = b"c1" + value = row.get_cells(column_family_id, column_id)[0].value.decode("utf-8") + + client.close() + + print("Row key: {}\nData: {}".format(row_key, value)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument( + "instance_id", help="ID of the Cloud Bigtable instance to connect to." + ) + parser.add_argument( + "--table", help="Existing table used in the quickstart.", default="my-table" + ) + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) + +# [END bigtable_quickstart_data_client] diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/main_test.py b/packages/google-cloud-bigtable/samples/data_client/quickstart/main_test.py new file mode 100644 index 000000000000..e7d82e5e1ffc --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/main_test.py @@ -0,0 +1,48 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid +from typing import Generator + +import pytest + +from google.cloud.bigtable.data import BigtableDataClient, SetCell + +from ..utils import create_table_cm +from .main import main + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"quickstart-test-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture +def table_id() -> Generator[str, None, None]: + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"cf1": None}): + _populate_table(TABLE_ID) + yield TABLE_ID + + +def _populate_table(table_id: str): + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + table.mutate_row("r1", SetCell("cf1", "c1", "test-value")) + + +def test_main(capsys, table_id): + main(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + assert "Row key: r1\nData: test-value\n" in out diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/noxfile.py b/packages/google-cloud-bigtable/samples/data_client/quickstart/noxfile.py new file mode 100644 index 000000000000..a2a659dcca04 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/noxfile.py @@ -0,0 +1,292 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +import sys +from pathlib import Path +from typing import Callable, Dict, Optional + +import nox + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8") + else: + session.install("flake8", "flake8-annotations") + + args = FLAKE8_COMMON_ARGS + [ + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# format = isort + black +# + + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( + "**/test_*.py", recursive=True + ) + test_list.extend(glob.glob("**/tests", recursive=True)) + + if len(test_list) == 0: + print("No tests found, skipping directory.") + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + elif "pytest-xdist" in packages: + concurrent_args.extend(["-n", "auto"]) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """Returns the root folder of the project.""" + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements-test.txt b/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements-test.txt new file mode 100644 index 000000000000..ee4ba018603b --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements-test.txt @@ -0,0 +1,2 @@ +pytest +pytest-asyncio diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements.txt b/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements.txt new file mode 100644 index 000000000000..730d25dec63f --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable==2.35.0 diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets.py b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets.py new file mode 100644 index 000000000000..5075830de240 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def write_simple(table): + # [START bigtable_write_simple] + from google.cloud.bigtable.data import BigtableDataClient, SetCell + + def write_simple(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + row_key = b"phone#4c410523#20190501" + + cell_mutation = SetCell(family_id, "connected_cell", 1) + wifi_mutation = SetCell(family_id, "connected_wifi", 1) + os_mutation = SetCell(family_id, "os_build", "PQ2A.190405.003") + + table.mutate_row(row_key, cell_mutation) + table.mutate_row(row_key, wifi_mutation) + table.mutate_row(row_key, os_mutation) + + # [END bigtable_write_simple] + write_simple(table.client.project, table.instance_id, table.table_id) + + +def write_batch(table): + # [START bigtable_writes_batch] + from google.cloud.bigtable.data import BigtableDataClient + from google.cloud.bigtable.data.exceptions import MutationsExceptionGroup + from google.cloud.bigtable.data.mutations import RowMutationEntry, SetCell + + def write_batch(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + try: + with table.mutations_batcher() as batcher: + mutation_list = [ + SetCell(family_id, "connected_cell", 1), + SetCell(family_id, "connected_wifi", 1), + SetCell(family_id, "os_build", "12155.0.0-rc1"), + ] + batcher.append( + RowMutationEntry("tablet#a0b81f74#20190501", mutation_list) + ) + batcher.append( + RowMutationEntry("tablet#a0b81f74#20190502", mutation_list) + ) + except MutationsExceptionGroup as e: + for sub_exception in e.exceptions: + failed_entry: RowMutationEntry = sub_exception.entry + cause: Exception = sub_exception.__cause__ + print( + f"Failed mutation: {failed_entry.row_key} with error: {cause!r}" + ) + + # [END bigtable_writes_batch] + write_batch(table.client.project, table.instance_id, table.table_id) + + +def write_increment(table): + # [START bigtable_write_increment] + from google.cloud.bigtable.data import BigtableDataClient + from google.cloud.bigtable.data.read_modify_write_rules import IncrementRule + + def write_increment(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + row_key = "phone#4c410523#20190501" + + increment_rule = IncrementRule( + family_id, "connected_wifi", increment_amount=-1 + ) + result_row = table.read_modify_write_row(row_key, increment_rule) + + cell = result_row[0] + print(f"{cell.row_key} value: {int(cell)}") + + # [END bigtable_write_increment] + write_increment(table.client.project, table.instance_id, table.table_id) + + +def write_conditional(table): + # [START bigtable_writes_conditional] + from google.cloud.bigtable.data import BigtableDataClient, SetCell, row_filters + + def write_conditional(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + row_key = "phone#4c410523#20190501" + + row_filter = row_filters.RowFilterChain( + filters=[ + row_filters.FamilyNameRegexFilter(family_id), + row_filters.ColumnQualifierRegexFilter("os_build"), + row_filters.ValueRegexFilter("PQ2A\\..*"), + ] + ) + + if_true = SetCell(family_id, "os_name", "android") + result = table.check_and_mutate_row( + row_key, + row_filter, + true_case_mutations=if_true, + false_case_mutations=None, + ) + if result is True: + print("The row os_name was set to android") + + # [END bigtable_writes_conditional] + write_conditional(table.client.project, table.instance_id, table.table_id) + + +def write_aggregate(table): + # [START bigtable_write_aggregate] + import time + + from google.cloud.bigtable.data import BigtableDataClient + from google.cloud.bigtable.data.exceptions import MutationsExceptionGroup + from google.cloud.bigtable.data.mutations import AddToCell, RowMutationEntry + + def write_aggregate(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + table = client.get_table(instance_id, table_id) + row_key = "unique_device_ids_1" + try: + with table.mutations_batcher() as batcher: + reading = AddToCell( + family="counters", + qualifier="odometer", + value=32304, + timestamp_micros=time.time_ns() // 1000, + ) + batcher.append( + RowMutationEntry(row_key.encode("utf-8"), [reading]) + ) + except MutationsExceptionGroup as e: + for sub_exception in e.exceptions: + failed_entry: RowMutationEntry = sub_exception.entry + cause: Exception = sub_exception.__cause__ + print( + f"Failed mutation for row {failed_entry.row_key!r} with error: {cause!r}" + ) + + # [END bigtable_write_aggregate] + write_aggregate(table.client.project, table.instance_id, table.table_id) + + +def read_row(table): + # [START bigtable_reads_row] + from google.cloud.bigtable.data import BigtableDataClient + + def read_row(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_key = "phone#4c410523#20190501" + row = table.read_row(row_key) + print(row) + + # [END bigtable_reads_row] + read_row(table.client.project, table.instance_id, table.table_id) + + +def read_row_partial(table): + # [START bigtable_reads_row_partial] + from google.cloud.bigtable.data import BigtableDataClient, row_filters + + def read_row_partial(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_key = "phone#4c410523#20190501" + col_filter = row_filters.ColumnQualifierRegexFilter(b"os_build") + + row = table.read_row(row_key, row_filter=col_filter) + print(row) + + # [END bigtable_reads_row_partial] + read_row_partial(table.client.project, table.instance_id, table.table_id) + + +def read_rows_multiple(table): + # [START bigtable_reads_rows] + from google.cloud.bigtable.data import BigtableDataClient, ReadRowsQuery + + def read_rows(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + query = ReadRowsQuery( + row_keys=[b"phone#4c410523#20190501", b"phone#4c410523#20190502"] + ) + for row in table.read_rows(query): + print(row) + + # [END bigtable_reads_rows] + read_rows(table.client.project, table.instance_id, table.table_id) + + +def read_row_range(table): + # [START bigtable_reads_row_range] + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + RowRange, + ) + + def read_row_range(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_range = RowRange( + start_key=b"phone#4c410523#20190501", + end_key=b"phone#4c410523#201906201", + ) + query = ReadRowsQuery(row_ranges=[row_range]) + + for row in table.read_rows(query): + print(row) + + # [END bigtable_reads_row_range] + read_row_range(table.client.project, table.instance_id, table.table_id) + + +def read_with_prefix(table): + # [START bigtable_reads_prefix] + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + RowRange, + ) + + def read_prefix(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + prefix = "phone#" + end_key = prefix[:-1] + chr(ord(prefix[-1]) + 1) + prefix_range = RowRange(start_key=prefix, end_key=end_key) + query = ReadRowsQuery(row_ranges=[prefix_range]) + + for row in table.read_rows(query): + print(row) + + # [END bigtable_reads_prefix] + read_prefix(table.client.project, table.instance_id, table.table_id) + + +def read_with_filter(table): + # [START bigtable_reads_filter] + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + def read_with_filter(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_filter = row_filters.ValueRegexFilter(b"PQ2A.*$") + query = ReadRowsQuery(row_filter=row_filter) + + for row in table.read_rows(query): + print(row) + + # [END bigtable_reads_filter] + read_with_filter(table.client.project, table.instance_id, table.table_id) + + +def execute_query(table): + # [START bigtable_execute_query] + from google.cloud.bigtable.data import BigtableDataClient + + def execute_query(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + query = ( + "SELECT _key, stats_summary['os_build'], " + "stats_summary['connected_cell'], " + "stats_summary['connected_wifi'] " + f"from `{table_id}` WHERE _key=@row_key" + ) + result = client.execute_query( + query, + instance_id, + parameters={"row_key": b"phone#4c410523#20190501"}, + ) + results = list(result) + print(results) + + # [END bigtable_execute_query] + execute_query(table.client.project, table.instance_id, table.table_id) diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets_test.py new file mode 100644 index 000000000000..fbdc1a69f6a7 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets_test.py @@ -0,0 +1,105 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +import pytest + +from ...utils import create_table_cm +from . import data_client_snippets as data_snippets + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"data-client-sync-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture(scope="session") +def column_family_config(): + from google.cloud.bigtable_admin_v2 import types + + int_aggregate_type = types.Type.Aggregate( + input_type=types.Type(int64_type={"encoding": {"big_endian_bytes": {}}}), + sum={}, + ) + + return { + "family": types.ColumnFamily(), + "stats_summary": types.ColumnFamily(), + "counters": types.ColumnFamily( + value_type=types.Type(aggregate_type=int_aggregate_type) + ), + } + + +@pytest.fixture(scope="session") +def table_id(column_family_config): + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, column_family_config): + yield TABLE_ID + + +@pytest.fixture +def table(table_id): + from google.cloud.bigtable.data import BigtableDataClient + + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + yield table + + +def test_write_simple(table): + data_snippets.write_simple(table) + + +def test_write_batch(table): + data_snippets.write_batch(table) + + +def test_write_increment(table): + data_snippets.write_increment(table) + + +def test_write_conditional(table): + data_snippets.write_conditional(table) + + +def test_write_aggregate(table): + data_snippets.write_aggregate(table) + + +def test_read_row(table): + data_snippets.read_row(table) + + +def test_read_row_partial(table): + data_snippets.read_row_partial(table) + + +def test_read_rows_multiple(table): + data_snippets.read_rows_multiple(table) + + +def test_read_row_range(table): + data_snippets.read_row_range(table) + + +def test_read_with_prefix(table): + data_snippets.read_with_prefix(table) + + +def test_read_with_filter(table): + data_snippets.read_with_filter(table) + + +def test_execute_query(table): + data_snippets.execute_query(table) diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets.py b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets.py new file mode 100644 index 000000000000..6955e1119832 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START bigtable_delete_from_column_data_client] +def delete_from_column(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + DeleteRangeFromColumn, + ) + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.mutate_row( + "phone#4c410523#20190501", + DeleteRangeFromColumn(family="cell_plan", qualifier=b"data_plan_01gb"), + ) + + table.close() + client.close() + + +# [END bigtable_delete_from_column_data_client] + + +# [START bigtable_delete_from_column_family_data_client] +def delete_from_column_family(project_id, instance_id, table_id): + from google.cloud.bigtable.data import BigtableDataClient, DeleteAllFromFamily + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.mutate_row("phone#4c410523#20190501", DeleteAllFromFamily("cell_plan")) + + table.close() + client.close() + + +# [END bigtable_delete_from_column_family_data_client] + + +# [START bigtable_delete_from_row_data_client] +def delete_from_row(project_id, instance_id, table_id): + from google.cloud.bigtable.data import BigtableDataClient, DeleteAllFromRow + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.mutate_row("phone#4c410523#20190501", DeleteAllFromRow()) + + table.close() + client.close() + + +# [END bigtable_delete_from_row_data_client] + + +# [START bigtable_streaming_and_batching_data_client] +def streaming_and_batching(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + DeleteRangeFromColumn, + ReadRowsQuery, + RowMutationEntry, + ) + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + with table.mutations_batcher() as batcher: + for row in table.read_rows(ReadRowsQuery(limit=10)): + batcher.append( + RowMutationEntry( + row.row_key, + DeleteRangeFromColumn( + family="cell_plan", qualifier=b"data_plan_01gb" + ), + ) + ) + + table.close() + client.close() + + +# [END bigtable_streaming_and_batching_data_client] + + +# [START bigtable_check_and_mutate_data_client] +def check_and_mutate(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + DeleteRangeFromColumn, + ) + from google.cloud.bigtable.data.row_filters import LiteralValueFilter + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.check_and_mutate_row( + "phone#4c410523#20190501", + predicate=LiteralValueFilter("PQ2A.190405.003"), + true_case_mutations=DeleteRangeFromColumn( + family="cell_plan", qualifier=b"data_plan_01gb" + ), + ) + + table.close() + client.close() + + +# [END bigtable_check_and_mutate_data_client] diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets_test.py new file mode 100644 index 000000000000..ebd193135d52 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets_test.py @@ -0,0 +1,259 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import uuid +from typing import Generator + +import pytest +from google.cloud._helpers import _microseconds_from_datetime + +from ...utils import create_table_cm +from . import deletes_snippets + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"mobile-time-series-deletes-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture(scope="module", autouse=True) +def table_id() -> Generator[str, None, None]: + with create_table_cm( + PROJECT, + BIGTABLE_INSTANCE, + TABLE_ID, + {"stats_summary": None, "cell_plan": None}, + verbose=False, + ): + _populate_table(TABLE_ID) + yield TABLE_ID + + +def _populate_table(table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + RowMutationEntry, + SetCell, + ) + + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = timestamp - datetime.timedelta(hours=1) + + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + with table.mutations_batcher() as batcher: + batcher.append( + RowMutationEntry( + "phone#4c410523#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.003", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "true", + _microseconds_from_datetime(timestamp_minus_hr), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "false", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.004", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190505", + [ + SetCell( + "stats_summary", + "connected_cell", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190401.002", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + + +def assert_output_match(capsys, expected): + out, _ = capsys.readouterr() + assert out == expected + + +def test_delete_from_column(capsys, table_id): + deletes_snippets.delete_from_column(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_delete_from_column_family(capsys, table_id): + deletes_snippets.delete_from_column_family(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_delete_from_row(capsys, table_id): + deletes_snippets.delete_from_row(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_streaming_and_batching(capsys, table_id): + deletes_snippets.streaming_and_batching(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_check_and_mutate(capsys, table_id): + deletes_snippets.check_and_mutate(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets.py b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets.py new file mode 100644 index 000000000000..9f00201ae25f --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets.py @@ -0,0 +1,389 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START bigtable_filters_limit_row_sample_data_client] +def filter_limit_row_sample(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.RowSampleFilter(0.75)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_row_sample_data_client] +# [START bigtable_filters_limit_row_regex_data_client] +def filter_limit_row_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.RowKeyRegexFilter(".*#20190501$".encode("utf-8")) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_row_regex_data_client] +# [START bigtable_filters_limit_cells_per_col_data_client] +def filter_limit_cells_per_col(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.CellsColumnLimitFilter(2)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_cells_per_col_data_client] +# [START bigtable_filters_limit_cells_per_row_data_client] +def filter_limit_cells_per_row(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.CellsRowLimitFilter(2)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_cells_per_row_data_client] +# [START bigtable_filters_limit_cells_per_row_offset_data_client] +def filter_limit_cells_per_row_offset(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.CellsRowOffsetFilter(2)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_cells_per_row_offset_data_client] +# [START bigtable_filters_limit_col_family_regex_data_client] +def filter_limit_col_family_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.FamilyNameRegexFilter("stats_.*$".encode("utf-8")) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_col_family_regex_data_client] +# [START bigtable_filters_limit_col_qualifier_regex_data_client] +def filter_limit_col_qualifier_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ColumnQualifierRegexFilter( + "connected_.*$".encode("utf-8") + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_col_qualifier_regex_data_client] +# [START bigtable_filters_limit_col_range_data_client] +def filter_limit_col_range(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ColumnRangeFilter( + "cell_plan", b"data_plan_01gb", b"data_plan_10gb", inclusive_end=False + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_col_range_data_client] +# [START bigtable_filters_limit_value_range_data_client] +def filter_limit_value_range(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ValueRangeFilter(b"PQ2A.190405", b"PQ2A.190406") + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_value_range_data_client] +# [START bigtable_filters_limit_value_regex_data_client] + + +def filter_limit_value_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ValueRegexFilter("PQ2A.*$".encode("utf-8")) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_value_regex_data_client] +# [START bigtable_filters_limit_timestamp_range_data_client] +def filter_limit_timestamp_range(project_id, instance_id, table_id): + import datetime + + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + end = datetime.datetime(2019, 5, 1) + + query = ReadRowsQuery(row_filter=row_filters.TimestampRangeFilter(end=end)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_timestamp_range_data_client] +# [START bigtable_filters_limit_block_all_data_client] +def filter_limit_block_all(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.BlockAllFilter(True)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_block_all_data_client] +# [START bigtable_filters_limit_pass_all_data_client] +def filter_limit_pass_all(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.PassAllFilter(True)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_pass_all_data_client] +# [START bigtable_filters_modify_strip_value_data_client] +def filter_modify_strip_value(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.StripValueTransformerFilter(True)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_modify_strip_value_data_client] +# [START bigtable_filters_modify_apply_label_data_client] +def filter_modify_apply_label(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.ApplyLabelFilter(label="labelled")) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_modify_apply_label_data_client] +# [START bigtable_filters_composing_chain_data_client] +def filter_composing_chain(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.RowFilterChain( + filters=[ + row_filters.CellsColumnLimitFilter(1), + row_filters.FamilyNameRegexFilter("cell_plan"), + ] + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_composing_chain_data_client] +# [START bigtable_filters_composing_interleave_data_client] +def filter_composing_interleave(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.RowFilterUnion( + filters=[ + row_filters.ValueRegexFilter("true"), + row_filters.ColumnQualifierRegexFilter("os_build"), + ] + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_composing_interleave_data_client] +# [START bigtable_filters_composing_condition_data_client] +def filter_composing_condition(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ConditionalRowFilter( + predicate_filter=row_filters.RowFilterChain( + filters=[ + row_filters.ColumnQualifierRegexFilter("data_plan_10gb"), + row_filters.ValueRegexFilter("true"), + ] + ), + true_filter=row_filters.ApplyLabelFilter(label="passed-filter"), + false_filter=row_filters.ApplyLabelFilter(label="filtered-out"), + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_composing_condition_data_client] + + +def print_row(row): + from google.cloud._helpers import _datetime_from_microseconds + + print("Reading data for {}:".format(row.row_key.decode("utf-8"))) + last_family = None + for cell in row.cells: + if last_family != cell.family: + print("Column Family {}".format(cell.family)) + last_family = cell.family + + labels = " [{}]".format(",".join(cell.labels)) if len(cell.labels) else "" + print( + "\t{}: {} @{}{}".format( + cell.qualifier.decode("utf-8"), + cell.value.decode("utf-8"), + _datetime_from_microseconds(cell.timestamp_micros), + labels, + ) + ) + print("") diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets_test.py new file mode 100644 index 000000000000..71aaa582cede --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets_test.py @@ -0,0 +1,373 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import inspect +import os +import uuid +from typing import Generator + +import pytest +from google.cloud._helpers import _microseconds_from_datetime + +from ...utils import create_table_cm +from . import filter_snippets +from .snapshots.snap_filters_test import snapshots + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"mobile-time-series-filters-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture(scope="module", autouse=True) +def table_id() -> Generator[str, None, None]: + with create_table_cm( + PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None} + ): + _populate_table(TABLE_ID) + yield TABLE_ID + + +def _populate_table(table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + RowMutationEntry, + SetCell, + ) + + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = timestamp - datetime.timedelta(hours=1) + + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + with table.mutations_batcher() as batcher: + batcher.append( + RowMutationEntry( + "phone#4c410523#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.003", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "true", + _microseconds_from_datetime(timestamp_minus_hr), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "false", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.004", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190505", + [ + SetCell( + "stats_summary", + "connected_cell", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190401.002", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + + +def test_filter_limit_row_sample(capsys, table_id): + filter_snippets.filter_limit_row_sample(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + assert "Reading data for" in out + + +def test_filter_limit_row_regex(capsys, table_id): + filter_snippets.filter_limit_row_regex(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_cells_per_col(capsys, table_id): + filter_snippets.filter_limit_cells_per_col(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_cells_per_row(capsys, table_id): + filter_snippets.filter_limit_cells_per_row(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_cells_per_row_offset(capsys, table_id): + filter_snippets.filter_limit_cells_per_row_offset( + PROJECT, BIGTABLE_INSTANCE, table_id + ) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_col_family_regex(capsys, table_id): + filter_snippets.filter_limit_col_family_regex(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_col_qualifier_regex(capsys, table_id): + filter_snippets.filter_limit_col_qualifier_regex( + PROJECT, BIGTABLE_INSTANCE, table_id + ) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_col_range(capsys, table_id): + filter_snippets.filter_limit_col_range(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_value_range(capsys, table_id): + filter_snippets.filter_limit_value_range(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_value_regex(capsys, table_id): + filter_snippets.filter_limit_value_regex(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_timestamp_range(capsys, table_id): + filter_snippets.filter_limit_timestamp_range(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_block_all(capsys, table_id): + filter_snippets.filter_limit_block_all(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_pass_all(capsys, table_id): + filter_snippets.filter_limit_pass_all(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_modify_strip_value(capsys, table_id): + filter_snippets.filter_modify_strip_value(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_modify_apply_label(capsys, table_id): + filter_snippets.filter_modify_apply_label(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_composing_chain(capsys, table_id): + filter_snippets.filter_composing_chain(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_composing_interleave(capsys, table_id): + filter_snippets.filter_composing_interleave(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_composing_condition(capsys, table_id): + filter_snippets.filter_composing_condition(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/noxfile.py b/packages/google-cloud-bigtable/samples/data_client/snippets/noxfile.py new file mode 100644 index 000000000000..a2a659dcca04 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/noxfile.py @@ -0,0 +1,292 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +import sys +from pathlib import Path +from typing import Callable, Dict, Optional + +import nox + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8") + else: + session.install("flake8", "flake8-annotations") + + args = FLAKE8_COMMON_ARGS + [ + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# format = isort + black +# + + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( + "**/test_*.py", recursive=True + ) + test_list.extend(glob.glob("**/tests", recursive=True)) + + if len(test_list) == 0: + print("No tests found, skipping directory.") + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + elif "pytest-xdist" in packages: + concurrent_args.extend(["-n", "auto"]) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """Returns the root folder of the project.""" + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/requirements-test.txt b/packages/google-cloud-bigtable/samples/data_client/snippets/requirements-test.txt new file mode 100644 index 000000000000..ee4ba018603b --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/requirements-test.txt @@ -0,0 +1,2 @@ +pytest +pytest-asyncio diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/requirements.txt b/packages/google-cloud-bigtable/samples/data_client/snippets/requirements.txt new file mode 100644 index 000000000000..730d25dec63f --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable==2.35.0 diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/__init__.py b/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/snap_filters_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/snap_filters_test.py new file mode 100644 index 000000000000..0547ddddd858 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/snap_filters_test.py @@ -0,0 +1,503 @@ +# -*- coding: utf-8 -*- +# this was previously implemented using the `snapshottest` package (https://goo.gl/zC4yUc), +# which is not compatible with Python 3.12. So we moved to a standard dictionary storing +# expected outputs for each test +from __future__ import unicode_literals + +snapshots = {} + +snapshots["test_filter_limit_row_regex"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_cells_per_col" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_cells_per_row" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_cells_per_row_offset" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_col_family_regex" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_col_qualifier_regex" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 + +""" + +snapshots["test_filter_limit_col_range"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_value_range" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_value_regex" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_timestamp_range" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 + +""" + +snapshots["test_filter_limit_block_all"] = "" + +snapshots["test_filter_limit_pass_all"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_modify_strip_value" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_modify_apply_label" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 [labelled] +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 [labelled] +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [labelled] + +""" + +snapshots["test_filter_composing_chain"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_composing_interleave" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_composing_condition" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 [filtered-out] +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 [filtered-out] +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 [filtered-out] + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 [filtered-out] + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [filtered-out] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [filtered-out] + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [passed-filter] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 [passed-filter] + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [passed-filter] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [passed-filter] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [passed-filter] + +""" diff --git a/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py b/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py index 13899a87425b..b689a9c35d86 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py @@ -26,7 +26,7 @@ import argparse -# [START bigtable_hw_imports] +# [START bigtable_hw_imports_legacy] from datetime import datetime, timezone from google.cloud import bigtable @@ -34,7 +34,7 @@ from ..utils import wait_for_table -# [END bigtable_hw_imports] +# [END bigtable_hw_imports_legacy] # use to avoid warnings row_filters @@ -42,14 +42,14 @@ def main(project_id, instance_id, table_id): - # [START bigtable_hw_connect] + # [START bigtable_hw_connect_legacy] # The client must be created with admin=True because it will create a # table. client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) - # [END bigtable_hw_connect] + # [END bigtable_hw_connect_legacy] - # [START bigtable_hw_create_table] + # [START bigtable_hw_create_table_legacy] print("Creating the {} table.".format(table_id)) table = instance.table(table_id) @@ -63,13 +63,13 @@ def main(project_id, instance_id, table_id): table.create(column_families=column_families) else: print("Table {} already exists.".format(table_id)) - # [END bigtable_hw_create_table] + # [END bigtable_hw_create_table_legacy] try: # let table creation complete wait_for_table(table) - # [START bigtable_hw_write_rows] + # [START bigtable_hw_write_rows_legacy] print("Writing some greetings to the table.") greetings = [b"Hello World!", b"Hello Cloud Bigtable!", b"Hello Python!"] rows = [] @@ -98,27 +98,27 @@ def main(project_id, instance_id, table_id): ) rows.append(row) table.mutate_rows(rows) - # [END bigtable_hw_write_rows] + # [END bigtable_hw_write_rows_legacy] - # [START bigtable_hw_create_filter] + # [START bigtable_hw_create_filter_legacy] # Create a filter to only retrieve the most recent version of the cell # for each column across entire row. row_filter = bigtable.row_filters.CellsColumnLimitFilter(1) - # [END bigtable_hw_create_filter] + # [END bigtable_hw_create_filter_legacy] - # [START bigtable_hw_get_with_filter] - # [START bigtable_hw_get_by_key] + # [START bigtable_hw_get_with_filter_legacy] + # [START bigtable_hw_get_by_key_legacy] print("Getting a single greeting by row key.") key = b"greeting0" row = table.read_row(key, row_filter) cell = row.cells[column_family_id.decode("utf-8")][column][0] print(cell.value.decode("utf-8")) - # [END bigtable_hw_get_by_key] - # [END bigtable_hw_get_with_filter] + # [END bigtable_hw_get_by_key_legacy] + # [END bigtable_hw_get_with_filter_legacy] - # [START bigtable_hw_scan_with_filter] - # [START bigtable_hw_scan_all] + # [START bigtable_hw_scan_with_filter_legacy] + # [START bigtable_hw_scan_all_legacy] print("Scanning for all greetings:") partial_rows = table.read_rows(filter_=row_filter) @@ -126,14 +126,14 @@ def main(project_id, instance_id, table_id): column_family_id_str = column_family_id.decode("utf-8") cell = row.cells[column_family_id_str][column][0] print(cell.value.decode("utf-8")) - # [END bigtable_hw_scan_all] - # [END bigtable_hw_scan_with_filter] + # [END bigtable_hw_scan_all_legacy] + # [END bigtable_hw_scan_with_filter_legacy] finally: - # [START bigtable_hw_delete_table] + # [START bigtable_hw_delete_table_legacy] print("Deleting the {} table.".format(table_id)) table.delete() - # [END bigtable_hw_delete_table] + # [END bigtable_hw_delete_table_legacy] if __name__ == "__main__": diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py index 7341bfc46f19..53f1bfbb1906 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py @@ -55,14 +55,14 @@ def run_instance_operations(project_id, instance_id, cluster_id): labels = {"prod-label": "prod-label"} instance = client.instance(instance_id, labels=labels) - # [START bigtable_check_instance_exists] + # [START bigtable_check_instance_exists_legacy] if not instance.exists(): print("Instance {} does not exist.".format(instance_id)) else: print("Instance {} already exists.".format(instance_id)) - # [END bigtable_check_instance_exists] + # [END bigtable_check_instance_exists_legacy] - # [START bigtable_create_prod_instance] + # [START bigtable_create_prod_instance_legacy] cluster = instance.cluster( cluster_id, location_id=location_id, @@ -76,27 +76,27 @@ def run_instance_operations(project_id, instance_id, cluster_id): # Ensure the operation completes. operation.result(timeout=480) print("\nCreated instance: {}".format(instance_id)) - # [END bigtable_create_prod_instance] + # [END bigtable_create_prod_instance_legacy] - # [START bigtable_list_instances] + # [START bigtable_list_instances_legacy] print("\nListing instances:") for instance_local in client.list_instances()[0]: print(instance_local.instance_id) - # [END bigtable_list_instances] + # [END bigtable_list_instances_legacy] - # [START bigtable_get_instance] + # [START bigtable_get_instance_legacy] print( "\nName of instance: {}\nLabels: {}".format( instance.display_name, instance.labels ) ) - # [END bigtable_get_instance] + # [END bigtable_get_instance_legacy] - # [START bigtable_get_clusters] + # [START bigtable_get_clusters_legacy] print("\nListing clusters...") for cluster in instance.list_clusters()[0]: print(cluster.cluster_id) - # [END bigtable_get_clusters] + # [END bigtable_get_clusters_legacy] def delete_instance(project_id, instance_id): @@ -111,14 +111,14 @@ def delete_instance(project_id, instance_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) - # [START bigtable_delete_instance] + # [START bigtable_delete_instance_legacy] print("\nDeleting instance") if not instance.exists(): print("Instance {} does not exist.".format(instance_id)) else: instance.delete() print("Deleted instance: {}".format(instance_id)) - # [END bigtable_delete_instance] + # [END bigtable_delete_instance_legacy] def add_cluster(project_id, instance_id, cluster_id): @@ -144,7 +144,7 @@ def add_cluster(project_id, instance_id, cluster_id): print("Instance {} does not exist.".format(instance_id)) else: print("\nAdding cluster to instance {}".format(instance_id)) - # [START bigtable_create_cluster] + # [START bigtable_create_cluster_legacy] print("\nListing clusters...") for cluster in instance.list_clusters()[0]: print(cluster.cluster_id) @@ -161,7 +161,7 @@ def add_cluster(project_id, instance_id, cluster_id): # Ensure the operation completes. operation.result(timeout=480) print("\nCluster created: {}".format(cluster_id)) - # [END bigtable_create_cluster] + # [END bigtable_create_cluster_legacy] def delete_cluster(project_id, instance_id, cluster_id): @@ -181,7 +181,7 @@ def delete_cluster(project_id, instance_id, cluster_id): instance = client.instance(instance_id) cluster = instance.cluster(cluster_id) - # [START bigtable_delete_cluster] + # [START bigtable_delete_cluster_legacy] print("\nDeleting cluster") if cluster.exists(): cluster.delete() @@ -189,7 +189,7 @@ def delete_cluster(project_id, instance_id, cluster_id): else: print("\nCluster {} does not exist.".format(cluster_id)) - # [END bigtable_delete_cluster] + # [END bigtable_delete_cluster_legacy] if __name__ == "__main__": diff --git a/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py b/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py index 1f89e6aacc15..3703d895794c 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py @@ -38,7 +38,7 @@ def get_cpu_load(bigtable_instance, bigtable_cluster): Returns: float: The most recent Cloud Bigtable CPU usage metric """ - # [START bigtable_cpu] + # [START bigtable_cpu_legacy] client = monitoring_v3.MetricServiceClient() cpu_query = query.Query( client, @@ -51,7 +51,7 @@ def get_cpu_load(bigtable_instance, bigtable_cluster): ) cpu = next(cpu_query.iter()) return cpu.points[0].value.double_value - # [END bigtable_cpu] + # [END bigtable_cpu_legacy] def get_storage_utilization(bigtable_instance, bigtable_cluster): @@ -60,7 +60,7 @@ def get_storage_utilization(bigtable_instance, bigtable_cluster): Returns: float: The most recent Cloud Bigtable storage utilization metric """ - # [START bigtable_metric_scaler_storage_utilization] + # [START bigtable_metric_scaler_storage_utilization_legacy] client = monitoring_v3.MetricServiceClient() utilization_query = query.Query( client, @@ -73,7 +73,7 @@ def get_storage_utilization(bigtable_instance, bigtable_cluster): ) utilization = next(utilization_query.iter()) return utilization.points[0].value.double_value - # [END bigtable_metric_scaler_storage_utilization] + # [END bigtable_metric_scaler_storage_utilization_legacy] def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): @@ -105,7 +105,7 @@ def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): # The number of nodes to change the cluster by. size_change_step = 3 - # [START bigtable_scale] + # [START bigtable_scale_legacy] bigtable_client = bigtable.Client(admin=True) instance = bigtable_client.instance(bigtable_instance) instance.reload() @@ -140,7 +140,7 @@ def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): current_node_count, new_node_count, response.name ) ) - # [END bigtable_scale] + # [END bigtable_scale_legacy] def main( diff --git a/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py b/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py index 50bfe639426c..a8bfd79a7b36 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START bigtable_quickstart] +# [START bigtable_quickstart_legacy] import argparse from google.cloud import bigtable @@ -54,4 +54,4 @@ def main(project_id="project-id", instance_id="instance-id", table_id="my-table" args = parser.parse_args() main(args.project_id, args.instance_id, args.table) -# [END bigtable_quickstart] +# [END bigtable_quickstart_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py index d62cfa3328b0..bc502e6f5037 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py @@ -55,7 +55,7 @@ def run_table_operations(project_id, instance_id, table_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) with create_table_cm(project_id, instance_id, table_id, verbose=False) as table: - # [START bigtable_list_tables] + # [START bigtable_list_tables_legacy] tables = instance.list_tables() print("Listing tables in current project...") if tables != []: @@ -63,9 +63,9 @@ def run_table_operations(project_id, instance_id, table_id): print(tbl.table_id) else: print("No table exists in current project...") - # [END bigtable_list_tables] + # [END bigtable_list_tables_legacy] - # [START bigtable_create_family_gc_max_age] + # [START bigtable_create_family_gc_max_age_legacy] print("Creating column family cf1 with with MaxAge GC Rule...") # Create a column family with GC policy : maximum age # where age = current time minus cell timestamp @@ -76,9 +76,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family1 = table.column_family("cf1", max_age_rule) column_family1.create() print("Created column family cf1 with MaxAge GC Rule.") - # [END bigtable_create_family_gc_max_age] + # [END bigtable_create_family_gc_max_age_legacy] - # [START bigtable_create_family_gc_max_versions] + # [START bigtable_create_family_gc_max_versions_legacy] print("Creating column family cf2 with max versions GC rule...") # Create a column family with GC policy : most recent N versions # where 1 = most recent version @@ -89,9 +89,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family2 = table.column_family("cf2", max_versions_rule) column_family2.create() print("Created column family cf2 with Max Versions GC Rule.") - # [END bigtable_create_family_gc_max_versions] + # [END bigtable_create_family_gc_max_versions_legacy] - # [START bigtable_create_family_gc_union] + # [START bigtable_create_family_gc_union_legacy] print("Creating column family cf3 with union GC rule...") # Create a column family with GC policy to drop data that matches # at least one condition. @@ -107,9 +107,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family3 = table.column_family("cf3", union_rule) column_family3.create() print("Created column family cf3 with Union GC rule") - # [END bigtable_create_family_gc_union] + # [END bigtable_create_family_gc_union_legacy] - # [START bigtable_create_family_gc_intersection] + # [START bigtable_create_family_gc_intersection_legacy] print("Creating column family cf4 with Intersection GC rule...") # Create a column family with GC policy to drop data that matches # all conditions @@ -125,9 +125,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family4 = table.column_family("cf4", intersection_rule) column_family4.create() print("Created column family cf4 with Intersection GC rule.") - # [END bigtable_create_family_gc_intersection] + # [END bigtable_create_family_gc_intersection_legacy] - # [START bigtable_create_family_gc_nested] + # [START bigtable_create_family_gc_nested_legacy] print("Creating column family cf5 with a Nested GC rule...") # Create a column family with nested GC policies. # Create a nested GC rule: @@ -148,9 +148,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family5 = table.column_family("cf5", nested_rule) column_family5.create() print("Created column family cf5 with a Nested GC rule.") - # [END bigtable_create_family_gc_nested] + # [END bigtable_create_family_gc_nested_legacy] - # [START bigtable_list_column_families] + # [START bigtable_list_column_families_legacy] print("Printing Column Family and GC Rule for all column families...") column_families = table.list_column_families() for column_family_name, gc_rule in sorted(column_families.items()): @@ -172,30 +172,30 @@ def run_table_operations(project_id, instance_id, table_id): # } # } # } - # [END bigtable_list_column_families] + # [END bigtable_list_column_families_legacy] print("Print column family cf1 GC rule before update...") print("Column Family: cf1") print(column_family1.to_pb()) - # [START bigtable_update_gc_rule] + # [START bigtable_update_gc_rule_legacy] print("Updating column family cf1 GC rule...") # Update the column family cf1 to update the GC rule column_family1 = table.column_family("cf1", column_family.MaxVersionsGCRule(1)) column_family1.update() print("Updated column family cf1 GC rule\n") - # [END bigtable_update_gc_rule] + # [END bigtable_update_gc_rule_legacy] print("Print column family cf1 GC rule after update...") print("Column Family: cf1") print(column_family1.to_pb()) - # [START bigtable_delete_family] + # [START bigtable_delete_family_legacy] print("Delete a column family cf2...") # Delete a column family column_family2.delete() print("Column family cf2 deleted successfully.") - # [END bigtable_delete_family] + # [END bigtable_delete_family_legacy] print( 'execute command "python tableadmin.py delete [project_id] \ @@ -220,7 +220,7 @@ def delete_table(project_id, instance_id, table_id): instance = client.instance(instance_id) table = instance.table(table_id) - # [START bigtable_delete_table] + # [START bigtable_delete_table_legacy] # Delete the entire table print("Checking if table {} exists...".format(table_id)) @@ -231,7 +231,7 @@ def delete_table(project_id, instance_id, table_id): print("Deleted {} table.".format(table_id)) else: print("Table {} does not exists.".format(table_id)) - # [END bigtable_delete_table] + # [END bigtable_delete_table_legacy] if __name__ == "__main__":