There are a few options already for wrapping calls to Elastic Search (
such as ElasticSearch.Net and NEST) however, I wanted to create an object oriented strongly type interface for some simple queries - (with a view to also being able to convert to this from a
CDSA "where clause").
My particular implementation of Elastic search, Azure Search, can be called using a restful API interface with OData and Lucene syntax. So my idea was then eventually any OO representation of a query can eventually then be boiled down to a URL with a querystring (similar to how CDSA converts OO clauses to SQL).
My first step was to create an abstraction around the concept of a "filter" that can be applied to a search:
public abstract class Filter
{
public abstract string GetQueryString();
}
Since we are most likely going to be dealing with collections of filters, I created an extension to provide easy access to this method:
internal static class FilterExtensions
{
public static string AsQueryString(this IEnumerable<Filter> filters)
{
return string.Join(" and ", filters.Select(f => f.GetQueryString()));
}
}
The first concrete implementation is always required by Azure Search, it's the API version you intend to work with:
internal sealed class ApiVersionFilter : Filter
{
public override string GetQueryString()
{
return $"api-version=2016-09-01";
}
}
Most fields you are querying will either be some filterable scalar value like strings and numbers or collection of these values in a collection column. I represented these types of query using two classes:
public class FieldValuesFilter : Filter
{
private IEnumerable<IFieldValue> selectedValues;
private string @operator;
public FieldValuesFilter(string fieldName, IEnumerable<IFieldValue> selectedValues)
: this(fieldName, "eq", selectedValues)
{
}
public FieldValuesFilter(string fieldName, string @operator, IEnumerable<IFieldValue> selectedValues)
{
this.FieldName = fieldName;
this.selectedValues = selectedValues;
this.@operator = @operator;
}
public string FieldName { get; private set; }
public override string GetQueryString()
{
return "(" + string.Join(" or ", this.selectedValues.Select(v => $"{this.FieldName} {this.@operator} {v.GetFormattedValue()}")) + ")";
}
}
public class ArrayValueFilter : Filter
{
private IEnumerable<IFieldValue> selectedValues;
private string @operator;
public ArrayValueFilter(string fieldName, IEnumerable<IFieldValue> selectedValues)
: this(fieldName, "any", selectedValues)
{
}
public ArrayValueFilter(string fieldName, string @operator, IEnumerable<IFieldValue> selectedValues)
{
this.FieldName = fieldName;
this.selectedValues = selectedValues;
this.@operator = @operator;
}
public string FieldName { get; private set; }
public override string GetQueryString()
{
return "(" + string.Join(" or ", this.selectedValues.Select(v => $"{this.FieldName}/{this.@operator}(t: t eq {v.GetFormattedValue()})")) + ")";
}
}
You will notice that to abstract the field type I use an interface for IFieldValue for passing in the filter values, this is because depending on whether the data type is a string or a number the formatting will change.
The interface and the two implementing classes are below:
public interface IFieldValue
{
object GetValue();
string GetFormattedValue();
}
public class StringFieldValue : IFieldValue
{
private string value;
public StringFieldValue(string value)
{
this.value = value;
}
public string GetFormattedValue()
{
return $"'{this.value}'";
}
public object GetValue()
{
return this.value;
}
}
public class IntegerFieldValue : IFieldValue
{
private int value;
public IntegerFieldValue(int value)
{
this.value = value;
}
public string GetFormattedValue()
{
return this.value.ToString();
}
public object GetValue()
{
return this.value;
}
}
You can create other filters, such as a "well known text" or "proximity" filter to query spatial data:
public class WktFilter : Filter
{
private string wkt;
public WktFilter(string wkt)
{
this.Wkt = wkt;
}
public string Wkt
{
get
{
return this.wkt;
}
set
{
this.wkt = value;
}
}
public override string GetQueryString()
{
return $"geo.intersects(location, geography'{this.Wkt}')";
}
}
public class ProximityFilter : Filter
{
private double latitude;
private double longitude;
private double radiusInMeters;
public ProximityFilter(double latitude, double longitude, double radiusInMeters)
{
this.latitude = latitude;
this.longitude = longitude;
this.radiusInMeters = radiusInMeters;
}
public override string GetQueryString()
{
// Azure Search works with KM not M, so div by 1000
return $"geo.distance(location, geography'POINT({this.longitude} {this.latitude})') le {this.radiusInMeters / 1000d}";
}
}
Now that we have the ability to create filters using C# classes and for more derivatives to be added if you more more strong typing, then we need a way of converting this to an Azure Search query. For simplicity I again started with a base class representing a given service which encapsulates the ability to convert a set of filters into a query string or post body and holds the config and endpoint for the index:
public abstract class SearchServiceBase
{
private readonly AzureServiceConfig config;
public SearchServiceBase(AzureServiceConfig config)
{
this.config = config;
}
public abstract string Api { get; }
public AzureServiceConfig Config
{
get
{
return this.config;
}
}
protected string GetQueryString(string name, IEnumerable<Filter> filters, int? top, int? skip)
{
// parse the input parameters
StringBuilder requestParameters = new StringBuilder();
// name param
if (!string.IsNullOrEmpty(name))
{
requestParameters.Append($"search={name}&");
}
// add API version by default
requestParameters.Append(new ApiVersionFilter().GetQueryString() + "&");
if (top.HasValue)
{
requestParameters.Append($"$top={top.Value.ToString()}&");
}
if (skip.HasValue)
{
requestParameters.Append($"$skip={skip.Value.ToString()}&");
}
if (skip.HasValue && skip.Value == 0)
{
requestParameters.Append($"$count=true&");
}
// filters could be none, one or many
if (filters != null && filters.Any())
{
requestParameters.Append($"$filter={filters.AsQueryString()}&");
}
// get the resource
return requestParameters.Length > 0 ? "?" + requestParameters.ToString().TrimEnd('&') : string.Empty;
}
protected string GetPostQueryString()
{
return "?" + new ApiVersionFilter().GetQueryString();
}
protected string GetPostBody(string name, IEnumerable<Filter> filters, int? top, int? skip)
{
// parse the input parameters
StringWriter sw = new StringWriter();
JsonTextWriter writer = new JsonTextWriter(sw);
// {
writer.WriteStartObject();
if (skip.HasValue && skip.Value == 0)
{
writer.WritePropertyName("count");
writer.WriteValue("true");
}
if (!string.IsNullOrEmpty(name))
{
writer.WritePropertyName("search");
writer.WriteValue(name);
writer.WritePropertyName("searchMode");
writer.WriteValue("all");
}
if (top.HasValue)
{
writer.WritePropertyName("top");
writer.WriteValue(top.Value.ToString());
}
if (skip.HasValue)
{
writer.WritePropertyName("skip");
writer.WriteValue(skip.Value.ToString());
}
// filters could be none, one or many
if (filters != null && filters.Any())
{
writer.WritePropertyName("filter");
writer.WriteValue(filters.AsQueryString()); // querystring is same format as POST property value
}
// }
writer.WriteEndObject();
return sw.ToString();
}
}
So an example subclass which consumes this functionality:
public class ExampleSearchService : SearchServiceBase
{
private readonly string indexName;
public ExampleSearchService(AzureServiceConfig config, string indexName)
: base(config)
{
this.indexName = indexName;
}
public override string Api
{
get
{
return $"indexes/{this.indexName}/docs/search";
}
}
public async Task<IEnumerable<ExampleSearchResult>> GetResultsAsync(string name, IEnumerable<Filter> filters, int maxResultCount, bool usePost = false)
{
using (var webApiClient = new AzureElasticSearchClient(this.Config))
{
webApiClient.Timeout = new TimeSpan(0, 20, 0);
var results = new List<ExampleSearchResult>();
int pageSize = 1000;
int pagesToRetrieve = 1;
int pagesRetrieved = 0;
// at least one page, but may be more..
while (pagesRetrieved < pagesToRetrieve)
{
HttpResponseMessage result = null;
if (usePost)
{
string requestUrl = $"{this.Api}{this.GetPostQueryString()}";
string requestBody = this.GetPostBody(name, filters, pageSize, pagesRetrieved * pageSize);
HttpContent content = new StringContent(requestBody, System.Text.Encoding.UTF8, "application/json");
// call the API and increment received count
result = await webApiClient.PostAsync(requestUrl, content).ConfigureAwait(false);
}
else
{
// build the query url from the filters
string requestUrl = $"{this.Api}{this.GetQueryString(name, filters, pageSize, pagesRetreived * pageSize)}";
result = await webApiClient.GetAsync(requestUrl).ConfigureAwait(false);
}
pagesRetrieved++;
// if it was successful, we can process it
if (result.IsSuccessStatusCode)
{
// parse the JSON response
var jResponse = JObject.Parse(await result.Content.ReadAsStringAsync().ConfigureAwait(false));
// if this was the first page, we can parse the item count
if (pagesRetrieved == 1)
{
int count = (int)jResponse.GetValue("@odata.count");
// check against max result count and throw exception if over that
if (maxResultCount > 0)
{
if (count > maxResultCount)
{
throw new InvalidOperationException($"Search result count of {count} was greater than the maximum of {maxResultCount}");
}
}
pagesToRetrieve = (int)Math.Ceiling((double)count / (double)pageSize);
}
// now get the value, which is the array of results
JArray jsonResults = (JArray)jResponse.GetValue("value");
// loop over the JSON array and deserialise each result obejct
foreach (var resultData in jsonResults)
{
var result = resultData.ToObject<ExampleSearchResult>();
results.Add(result);
}
}
}
return results;
}
}
}
You will notice I created a wrapper for HttpClient, this is simply to encapsulate adding the base address and API key:
internal class AzureElasticSearchClient : HttpClient
{
public AzureElasticSearchClient(AzureServiceConfig config)
{
this.BaseAddress = new Uri($"https://{config.AzureSearchServiceName}.search.windows.net/");
this.DefaultRequestHeaders.Add("api-key", config.AzureSearchApiKey);
}
}
Here are some examples of how to create and add filters then call the example search service:
[TestFixture]
public class TestExampleSearchService
{
private ExampleSearchService sut = new ExampleSearchService();
[Test]
public async Task TestGetResultsInWktWithAdditionalFilter_Elastic()
{
// arrange
var testPolygonWkt = "POLYGON ((-1.6259765625 53.74404116282134, -1.6005706787109375 53.76089000834015, -1.5696716308593748 53.73876182109416, -1.6036605834960935 53.72799803200196, -1.6259765625 53.74404116282134))";
var polyFilter = new WktFilter(testPolygonWkt);
var examplePropertyFilter = new FieldValuesFilter("SomeIntProperty", new IntegerFieldValue[] { new IntegerFieldValue(1) });
// act
var startTime = DateTime.Now;
var result = await this.sut.GetResultsAsync(string.Empty, new Filters.Filter[] { polyFilter, examplePropertyFilter });
Trace.WriteLine($"Time taken: {DateTime.Now.Subtract(startTime).TotalMilliseconds} ms");
Trace.WriteLine($"# recs {result.Count()}");
// assert
Assert.Greater(result.Count(), 0);
}
[Test]
public async Task TestGetResultsInProximityWithName_Elastic()
{
// arrange
var longi = -1.6259765625;
var lati = 53.74404116282134;
var dist = 200;
var proxFilter = new ProximityFilter(lati, longi, dist);
// act
var startTime = DateTime.Now;
var result = await this.sut.GetResultsAsync("example name", new Filters.Filter[] { proxFilter });
Trace.WriteLine($"# recs {result.Count()}");
Trace.WriteLine($"Time taken: {DateTime.Now.Subtract(startTime).TotalMilliseconds} ms");
// assert
Assert.Greater(result.Count(), 0);
}
[Test]
public async Task TestGetResultsInArrayFilter_Elastic()
{
// arrange
var possibleValues = {"hello", "world"}.Select(s => new StringFieldValue(s));
var exampleArrayPropertyFilter = new FieldValuesFilter("SomeStringArrayProperty", possibleValues);
// act
var startTime = DateTime.Now;
var result = await this.sut.GetResultsAsync(string.Empty, new Filters.Filter[] { exampleArrayPropertyFilter });
Trace.WriteLine($"Time taken: {DateTime.Now.Subtract(startTime).TotalMilliseconds} ms");
Trace.WriteLine($"# recs {result.Count()}");
// assert
Assert.Greater(result.Count(), 0);
}
}
With this structure all now in place, I wanted to plug this into an existing CDSA application which was currently using SQL for performing queries. I therefore needed a way to convert from traditional "CDSA WhereClause" objects to my new Azure Search filter structure. I created a basic implementation, which isn't 100% compatible with all clauses yet, but for most use cases it works fine:
internal class WhereClauseParser
{
Regex isNumeric = new Regex("^\\d+$");
public IEnumerable<FieldValuesFilter> ParseWhereClause(WhereClause clause)
{
if (clause == null || clause.RecursiveClauseList.Count == 0)
{
return null;
}
return this.ParseWhereClauseWithSubgroups(clause);
}
private IEnumerable<FieldValuesFilter> ParseWhereClauseWithSubgroups(WhereClause clause)
{
if (clause.ConjunctionOperator == ConjunctionOperator.And)
{
var myFilters = new List<FieldValuesFilter>();
foreach (var whereClauseElement in clause.ClauseList)
{
myFilters.Add(this.ParseWhereClauseElement(whereClauseElement));
}
if (clause.SubGroups.Count > 0)
{
foreach (var subClause in clause.SubGroups)
{
myFilters.AddRange(this.ParseWhereClauseWithSubgroups(subClause));
}
}
return myFilters;
}
else
{
throw new NotImplementedException("Elastic search clause parser currently only supports the 'AND' conjunction.");
}
}
private FieldValuesFilter ParseWhereClauseElement(WhereClauseElement whereClauseElement)
{
// start with the defaults
var fieldName = whereClauseElement.CompareItem;
var values = new object[] { whereClauseElement.CompareValue };
var @operator = string.Empty;
// don't need qualified paths, remove the dots
fieldName = fieldName.Replace(".", "");
switch (whereClauseElement.Operator)
{
case Operator.Equals:
case Operator.Like: // todo: this should be a wildcard search, not an exact equality
@operator = "eq";
break;
case Operator.NotEqual:
@operator = "ne";
break;
case Operator.GreaterThan:
@operator = "gt";
break;
case Operator.LessThan:
@operator = "lt";
break;
case Operator.GreaterThanEqualTo:
@operator = "ge";
break;
case Operator.LessThanEqualTo:
@operator = "le";
break;
case Operator.In:
@operator = "eq";
values = values[0].ToString().Split(',');
// if it was an array of numbers stored as CSV, then unpack the numbers as ints.
if (values.All(v => this.isNumeric.IsMatch(v.ToString())))
{
values = values.Select(v => (object) int.Parse(v.ToString())).ToArray();
}
break;
default:
throw new NotImplementedException("Elastic search clause parser currently does not support this operator");
}
return new FieldValuesFilter(fieldName, @operator, values.Select(v => this.FieldValueFactory(v)));
}
private IFieldValue FieldValueFactory(object v)
{
// a dirty type check, lets call it a "factory"
if (v is int)
{
return new IntegerFieldValue((int)v);
}
else
{
return new StringFieldValue(v.ToString());
}
}
}
From the beginning of 2018 I have taken the leap into self-employment. I am now working as a consultant/contract software developer through