feat(storage): setting idempotency token header

google-apis-core/lib/google/apis/core/api_command.rb

@@ -18,7 +18,7 @@
 require 'google/apis/errors'
 require 'json'
 require 'retriable'
-require "securerandom"
+require 'securerandom'
 
 module Google
   module Apis
@@ -71,6 +71,7 @@ def initialize(method, url, body: nil, client_version: nil)
         def prepare!
           set_api_client_header
           set_user_project_header
+          set_idempotency_token_header if options&.add_idempotency_token_header
           if options&.api_format_version
             header['X-Goog-Api-Format-Version'] = options.api_format_version.to_s
           end
@@ -151,7 +152,7 @@ def set_api_client_header
             .map { |(a, b)| b }
             .join(' ')
             .split
-            .find_all { |s| s !~ %r{^gl-ruby/|^gdcl/} }
+            .find_all { |s| s !~ %r{^gl-ruby/|^gdcl/|^gccl-invocation-id/} }
             .join(' ')
           # Report 0.x.y versions that are in separate packages as 1.x.y.
           # Thus, reported gdcl/0.x.y versions are monopackage clients, while
@@ -175,8 +176,17 @@ def set_user_project_header
           header['X-Goog-User-Project'] = quota_project_id if quota_project_id
         end
 
+        def set_idempotency_token_header
+          return if options&.header&.any? { |k, _| k.to_s.downcase == 'x-goog-gcs-idempotency-token' }
+          return if header.any? { |k, _| k.to_s.downcase == 'x-goog-gcs-idempotency-token' }
+
+          @idempotency_token ||= SecureRandom.uuid
+          header['X-Goog-Gcs-Idempotency-Token'] = @idempotency_token
+        end
+
         def invocation_id_header
-          "gccl-invocation-id/#{SecureRandom.uuid}"
+          @invocation_id ||= SecureRandom.uuid
+          "gccl-invocation-id/#{@invocation_id}"
         end
 
         # Attempt to parse a JSON error message

google-apis-core/lib/google/apis/options.rb

@@ -41,7 +41,8 @@ module Apis
       :quota_project,
       :query,
       :add_invocation_id_header,
-      :upload_chunk_size
+      :upload_chunk_size,
+      :add_idempotency_token_header
     )
 
     # General client options
@@ -104,6 +105,8 @@ class RequestOptions
       #   @return [Boolean] True if the header gccl-invocation-id need to be set
       # @!attribute [rw] upload_chunk_size
       #   @return [Integer] The chunk size of storage upload. The default value is 100 MB.
+      # @!attribute [rw] add_idempotency_token_header
+      #   @return [Boolean] Flag to control whether the X-Goog-Gcs-Idempotency-Token is sent.
 
       # Get the default options
       # @return [Google::Apis::RequestOptions]
@@ -140,5 +143,6 @@ def merge(options)
     RequestOptions.default.quota_project = nil
     RequestOptions.default.add_invocation_id_header = false
     RequestOptions.default.upload_chunk_size = 100 * 1024 * 1024 # 100 MB
+    RequestOptions.default.add_idempotency_token_header = true
   end
 end

google-apis-core/spec/google/apis/core/api_command_spec.rb

@@ -34,7 +34,6 @@
 
   let(:client_version) { "1.2.3" }
   let(:x_goog_api_client_value) { "gl-ruby/#{RUBY_VERSION} gdcl/#{client_version}" }
-
   context('with preparation') do
     let(:command) do
       Google::Apis::Core::ApiCommand.new(:get, 'https://www.googleapis.com/zoo/animals', client_version: client_version)
@@ -67,6 +66,14 @@
       expect(command.header['x-goog-api-client']).to be nil
     end
 
+    it 'should not accumulate duplicate gccl-invocation-id clauses when prepare! is called multiple times' do
+      command.options.add_invocation_id_header = true
+      command.prepare!
+      first_header = command.header['X-Goog-Api-Client']
+      command.prepare!
+      expect(command.header['X-Goog-Api-Client']).to eql first_header
+    end
+
     it 'should not set the X-Goog-User-Project header if there is no quota_project' do
       command.prepare!
       expect(command.header['X-Goog-User-Project']).to be_nil
@@ -88,7 +95,36 @@
     it "should set the gccl-invocation-id to a random UUID" do
       command.options.add_invocation_id_header = true
       command.prepare!
-      expect(command.header["X-Goog-Api-Client"]).to include("gccl-invocation-id")
+      expect(command.header["X-Goog-Api-Client"]).to match(/gccl-invocation-id\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/)
+    end
+
+    it "should set the X-Goog-Gcs-Idempotency-Token header" do
+      command.prepare!
+      expect(command.header['X-Goog-Gcs-Idempotency-Token']).not_to be_nil
+      expect(command.header['X-Goog-Gcs-Idempotency-Token']).to match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/)
+    end
+
+    it "should not set the X-Goog-Gcs-Idempotency-Token header when option is false" do
+      command.options.add_idempotency_token_header = false
+      command.prepare!
+      expect(command.header['X-Goog-Gcs-Idempotency-Token']).to be_nil
+    end
+
+    it 'should generate different tokens for different command instances' do
+      cmd1 = Google::Apis::Core::ApiCommand.new(:get, 'https://www.googleapis.com/zoo/animals')
+      cmd2 = Google::Apis::Core::ApiCommand.new(:get, 'https://www.googleapis.com/zoo/animals')
+
+      cmd1.prepare!
+      cmd2.prepare!
+
+      expect(cmd1.header['X-Goog-Gcs-Idempotency-Token']).not_to eq(cmd2.header['X-Goog-Gcs-Idempotency-Token'])
+    end
+
+    it "should respect custom X-Goog-Gcs-Idempotency-Token in options.header regardless of casing" do
+      command.options.header = { 'x-goog-gcs-idempotency-token' => 'my-custom-token' }
+      command.prepare!
+      expect(command.header['x-goog-gcs-idempotency-token']).to eql 'my-custom-token'
+      expect(command.header['X-Goog-Gcs-Idempotency-Token']).to be_nil
     end
   end
 
@@ -250,11 +286,18 @@
       command.options.add_invocation_id_header = true
       result = command.execute(client)
       invocation_id_header = command.header["X-Goog-Api-Client"]
-
       expect(invocation_id_header).to include("gccl-invocation-id")
       expect(a_request(:get, 'https://www.googleapis.com/zoo/animals')
         .with { |req| req.headers['X-Goog-Api-Client'] == invocation_id_header }).to have_been_made.times(2)
     end
+
+    it 'should keep same idempotency_token across retries' do
+      result = command.execute(client)
+      idempotency_token_header = command.header['X-Goog-Gcs-Idempotency-Token']
+      expect(a_request(:get, 'https://www.googleapis.com/zoo/animals')
+        .with { |req| req.headers['X-Goog-Gcs-Idempotency-Token'] == idempotency_token_header })
+        .to have_been_made.times(2)
+    end
   end
 
   context('with a project not linked response') do

google-apis-core/spec/google/apis/core/service_spec.rb

@@ -421,7 +421,6 @@
 
   context 'with batch uploads' do
     before(:example) do
-      allow(SecureRandom).to receive(:uuid).and_return('b1981e17-f622-49af-b2eb-203308b1b17d')
       allow(Digest::SHA1).to receive(:hexdigest).and_return('outer', 'inner')
       response = <<EOF.gsub(/\n/, "\r\n")
 --batch123
@@ -471,12 +470,13 @@
       expected_body = <<EOF.gsub(/\n/, "\r\n")
 --outer
 Content-Type: application/http
-Content-Id: <b1981e17-f622-49af-b2eb-203308b1b17d\\+0>
+Content-Id: <[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\+0>
 Content-Length: \\d+
 Content-Transfer-Encoding: binary
 
 POST /upload/zoo/animals\\? HTTP/1\\.1
 X-Goog-Api-Client: #{Regexp.escape(x_goog_api_client_value)}
+X-Goog-Gcs-Idempotency-Token: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
 Content-Type: multipart/related; boundary=inner
 X-Goog-Upload-Protocol: multipart
 Authorization: Bearer a token

google-apis-core/spec/google/apis/options_spec.rb

@@ -52,5 +52,6 @@
     expect(defaults.use_opencensus).to be true
     expect(defaults.quota_project).to be_nil
     expect(defaults.upload_chunk_size).to eq(100 * 1024 * 1024)
+    expect(defaults.add_idempotency_token_header).to be true
  end
 end