Skip to content

Commit

Permalink
Very early prototype of ExplainOptions for Datastore
Browse files Browse the repository at this point in the history
This is just an attempt at the shape of things.

Still to do:

- Naming!
- Documentation
- Testing
- We can refactor a lot of other code to use AdvancedQuery
- Aggregation queries
- Transactions (hopefully put transaction ID support into AdvancedQuery and
  then just add overloads)
  • Loading branch information
jskeet committed Jan 10, 2025
1 parent fa011c0 commit 2caa892
Show file tree
Hide file tree
Showing 9 changed files with 275 additions and 46 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,58 @@ public void SyncQueries()
});
}

[Fact]
public void Explain()
{
var db = _fixture.CreateDatastoreDb();
var keyFactory = db.CreateKeyFactory("syncqueries_explain");
using (var transaction = db.BeginTransaction())
{
var entities = Enumerable.Range(0, 5)
.Select(x => new Entity { Key = keyFactory.CreateIncompleteKey(), ["x"] = x })
.ToList();
transaction.Insert(entities);
transaction.Commit();
}

var query = new Query("syncqueries") { Filter = Filter.LessThan("x", 3) };
var gql = new GqlQuery { QueryString = "SELECT * FROM syncqueries WHERE x < 3", AllowLiterals = true };

_fixture.RetryQuery(() =>
{
var results = db.RunQuery(new AdvancedQuery { GqlQuery = gql, ExplainOptions = new ExplainOptions { Analyze = false } });
Assert.NotNull(results.PlanSummary);
Assert.Null(results.ExecutionStats);
Assert.Empty(results.Entities);
});
}

[Fact]
public void ExplainAnalyze()
{
var db = _fixture.CreateDatastoreDb();
var keyFactory = db.CreateKeyFactory("syncqueries_explain");
using (var transaction = db.BeginTransaction())
{
var entities = Enumerable.Range(0, 5)
.Select(x => new Entity { Key = keyFactory.CreateIncompleteKey(), ["x"] = x })
.ToList();
transaction.Insert(entities);
transaction.Commit();
}

var query = new Query("syncqueries") { Filter = Filter.LessThan("x", 3) };
var gql = new GqlQuery { QueryString = "SELECT * FROM syncqueries WHERE x < 3", AllowLiterals = true };

_fixture.RetryQuery(() =>
{
var results = db.RunQuery(new AdvancedQuery { GqlQuery = gql, ExplainOptions = new ExplainOptions { Analyze = true } });
Assert.NotNull(results.PlanSummary);
Assert.NotNull(results.ExecutionStats);
Assert.NotEmpty(results.Entities);
});
}

[Fact]
public async Task AsyncQueries()
{
Expand Down
78 changes: 42 additions & 36 deletions apis/Google.Cloud.Datastore.V1/Google.Cloud.Datastore.V1.sln
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,53 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.2.32516.85
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1", "Google.Cloud.Datastore.V1\Google.Cloud.Datastore.V1.csproj", "{8D442A59-2660-609D-33A9-844FA37132AA}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1", "Google.Cloud.Datastore.V1\Google.Cloud.Datastore.V1.csproj", "{8D442A59-2660-609D-33A9-844FA37132AA}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.Snippets", "Google.Cloud.Datastore.V1.Snippets\Google.Cloud.Datastore.V1.Snippets.csproj", "{B05695A0-022D-652A-8FBE-BD5976085F58}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.Snippets", "Google.Cloud.Datastore.V1.Snippets\Google.Cloud.Datastore.V1.Snippets.csproj", "{B05695A0-022D-652A-8FBE-BD5976085F58}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.GeneratedSnippets", "Google.Cloud.Datastore.V1.GeneratedSnippets\Google.Cloud.Datastore.V1.GeneratedSnippets.csproj", "{D71882DB-B962-6046-E4D2-76BFADC6C0C4}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.GeneratedSnippets", "Google.Cloud.Datastore.V1.GeneratedSnippets\Google.Cloud.Datastore.V1.GeneratedSnippets.csproj", "{D71882DB-B962-6046-E4D2-76BFADC6C0C4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.IntegrationTests", "Google.Cloud.Datastore.V1.IntegrationTests\Google.Cloud.Datastore.V1.IntegrationTests.csproj", "{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.IntegrationTests", "Google.Cloud.Datastore.V1.IntegrationTests\Google.Cloud.Datastore.V1.IntegrationTests.csproj", "{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.Tests", "Google.Cloud.Datastore.V1.Tests\Google.Cloud.Datastore.V1.Tests.csproj", "{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.Tests", "Google.Cloud.Datastore.V1.Tests\Google.Cloud.Datastore.V1.Tests.csproj", "{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.ClientTesting", "..\..\tools\Google.Cloud.ClientTesting\Google.Cloud.ClientTesting.csproj", "{29974B0C-A7B0-8CA8-AE32-99F622C89044}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.ClientTesting", "..\..\tools\Google.Cloud.ClientTesting\Google.Cloud.ClientTesting.csproj", "{29974B0C-A7B0-8CA8-AE32-99F622C89044}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.Build.0 = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.Build.0 = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.Build.0 = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.Build.0 = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.Build.0 = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.Build.0 = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.Build.0 = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.Build.0 = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.Build.0 = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.Build.0 = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {69E8BF28-102E-4CFC-BB48-6543F4E335CC}
EndGlobalSection
EndGlobal
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License"):
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using static Google.Cloud.Datastore.V1.ReadOptions.Types;

namespace Google.Cloud.Datastore.V1;

/// <summary>
/// Prototype only!
/// </summary>
public class AdvancedQuery
{
/// <summary>
///
/// </summary>
public GqlQuery GqlQuery { get; set; }

/// <summary>
///
/// </summary>
public Query Query { get; set; }

/// <summary>
///
/// </summary>
public ExplainOptions ExplainOptions { get; set; }

/// <summary>
///
/// </summary>
public ReadConsistency? ReadConsistency { get; set; }

internal RunQueryRequest ToRequest(string projectId, string databaseId, PartitionId partitionId)
{
var request = new RunQueryRequest
{
ProjectId = projectId,
DatabaseId = databaseId,
PartitionId = partitionId,
ReadOptions = GetReadOptions(ReadConsistency),
ExplainOptions = ExplainOptions
};
// TODO: Validation of exactly one being set, or make the properties behave like a oneof.
if (GqlQuery is not null)
{
request.GqlQuery = GqlQuery;
}
if (Query is not null)
{
request.Query = Query;
}
return request;
}

private static ReadOptions GetReadOptions(ReadConsistency? readConsistency) =>
readConsistency == null ? null : new ReadOptions { ReadConsistency = readConsistency.Value };
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2016 Google Inc. All Rights Reserved.
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -18,6 +18,7 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using static Google.Cloud.Datastore.V1.QueryResultBatch.Types;

namespace Google.Cloud.Datastore.V1
{
Expand Down Expand Up @@ -63,9 +64,10 @@ public async Task<DatastoreQueryResults> GetAllResultsAsync()
// cursors etc, but that's likely to be insignificant compared with the
// entity data, and is immediately eligible for garbage collection.
var responses = await _responses.ToListAsync().ConfigureAwait(false);
var entities = responses.SelectMany(r => r.Batch.EntityResults.Select(er => er.Entity)).ToList().AsReadOnly();
var entities = responses.SelectMany(r => r.Batch?.EntityResults.Select(er => er.Entity) ?? Enumerable.Empty<Entity>()).ToList().AsReadOnly();
var lastBatch = responses.Last().Batch;
return new DatastoreQueryResults(entities, lastBatch.MoreResults, lastBatch.EndCursor);
var metrics = responses.Last().ExplainMetrics;
return new DatastoreQueryResults(entities, lastBatch?.MoreResults ?? MoreResultsType.Unspecified, lastBatch?.EndCursor, metrics);
}

/// <inheritdoc />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,18 @@ public virtual KeyFactory CreateKeyFactory(string kind)
throw new NotImplementedException();
}

/// <summary>
/// Runs the given query eagerly, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
/// number of results, for example to build a web application which fetches results in pages.
/// </summary>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>The complete query results.</returns>
public virtual DatastoreQueryResults RunQuery(
AdvancedQuery query, CallSettings callSettings = null) =>
RunQueryLazily(query, callSettings).GetAllResults();

/// <summary>
/// Runs the given query eagerly, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
Expand All @@ -115,6 +127,17 @@ public virtual DatastoreQueryResults RunQuery(
Query query, ReadConsistency? readConsistency = null, CallSettings callSettings = null) =>
RunQueryLazily(query, readConsistency, callSettings).GetAllResults();

/// <summary>
/// Runs the given query eagerly and asynchronously, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
/// number of results, for example to build a web application which fetches results in pages.
/// </summary>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>A task representing the asynchronous operation. The result of the task is the complete set of query results.</returns>
public virtual Task<DatastoreQueryResults> RunQueryAsync(AdvancedQuery query, CallSettings callSettings = null) =>
RunQueryLazilyAsync(query, callSettings).GetAllResultsAsync();

/// <summary>
/// Runs the given query eagerly and asynchronously, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
Expand Down Expand Up @@ -176,6 +199,23 @@ public virtual Task<AggregationQueryResults> RunAggregationQueryAsync(GqlQuery q
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query.
/// </summary>
/// <remarks>
/// The results are requested lazily: no API calls will be made until the application starts
/// iterating over the results. Iterating over the same <see cref="LazyDatastoreQuery"/> object
/// multiple times will execute the query again, potentially returning different results.
/// </remarks>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>A <see cref="LazyDatastoreQuery"/> representing the lazy query results.</returns>
public virtual LazyDatastoreQuery RunQueryLazily(
AdvancedQuery query, CallSettings callSettings = null)
{
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query.
/// </summary>
Expand All @@ -194,6 +234,22 @@ public virtual LazyDatastoreQuery RunQueryLazily(
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query for asynchronous consumption.
/// </summary>
/// <remarks>
/// The results are requested lazily: no API calls will be made until the application starts
/// iterating over the results. Iterating over the same <see cref="LazyDatastoreQuery"/> object
/// multiple times will execute the query again, potentially returning different results.
/// </remarks>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>An <see cref="AsyncLazyDatastoreQuery"/> representing the lazy query results.</returns>
public virtual AsyncLazyDatastoreQuery RunQueryLazilyAsync(AdvancedQuery query, CallSettings callSettings = null)
{
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query for asynchronous consumption.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
using Google.Api.Gax;
using Google.Api.Gax.Grpc;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using static Google.Cloud.Datastore.V1.CommitRequest.Types;
Expand Down Expand Up @@ -215,6 +217,24 @@ public override AsyncLazyDatastoreQuery RunQueryLazilyAsync(
return new AsyncLazyDatastoreQuery(streamer.Async());
}

/// <inheritdoc/>
public override LazyDatastoreQuery RunQueryLazily(AdvancedQuery query, CallSettings callSettings = null)
{
GaxPreconditions.CheckNotNull(query, nameof(query));
var request = query.ToRequest(ProjectId, DatabaseId, _partitionId);
var streamer = new QueryStreamer(request, Client.RunQueryApiCall, callSettings);
return new LazyDatastoreQuery(streamer.Sync());
}

/// <inheritdoc/>
public override AsyncLazyDatastoreQuery RunQueryLazilyAsync(AdvancedQuery query, CallSettings callSettings = null)
{
GaxPreconditions.CheckNotNull(query, nameof(query));
var request = query.ToRequest(ProjectId, DatabaseId, _partitionId);
var streamer = new QueryStreamer(request, Client.RunQueryApiCall, callSettings);
return new AsyncLazyDatastoreQuery(streamer.Async());
}

/// <inheritdoc/>
public override DatastoreTransaction BeginTransaction(CallSettings callSettings = null)
{
Expand Down
Loading

0 comments on commit 2caa892

Please sign in to comment.