AS1024/GeoFeed
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Inital work on breaking out the core logic out of the MVC API Web App in preparation to migrating to minimal API's in a seperate project

Browse Source
This commit is contained in:
Jeff Leung
2024-01-13 12:42:46 -08:00
parent 52708bd2c2
commit bdd247e6e0
18 changed files with 91 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
AS1024.GeoFeed/AS1024.GeoFeed.csproj
View File

@@ -21,4 +21,9 @@
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\AS1024.GeoFeed.Core\AS1024.GeoFeed.Core.csproj" />
<ProjectReference Include="..\AS1024.GeoFeed.Models\AS1024.GeoFeed.Models.csproj" />
</ItemGroup>
</Project>

6
AS1024.GeoFeed/Controllers/GeofeedController.cs
View File

@@ -1,10 +1,10 @@
using Microsoft.AspNetCore.Mvc;
using AS1024.GeoFeed.Interfaces;
using AS1024.GeoFeed.GeoFeedBuilder;
using AS1024.GeoFeed.Core.Interfaces;
using Microsoft.Extensions.Caching.Memory;
using AS1024.GeoFeed.Models;
using System.Text;
using AS1024.GeoFeed.GeoFeedLocalCache;
using AS1024.GeoFeed.Core.GeoFeedLocalCache;
using AS1024.GeoFeed.Core.Tools;
namespace AS1024.GeoFeed.Controllers
{

20
AS1024.GeoFeed/GeoFeedBuilder/GeoFeedTools.cs
View File

@@ -1,20 +0,0 @@
using AS1024.GeoFeed.Models;
using System.Text;
namespace AS1024.GeoFeed.GeoFeedBuilder
{
public static class GeoFeedTools
{
public static string ToGeoFeedCsv(this List<IPGeoFeed> geoFeeds)
{
StringBuilder csvContent = new();
foreach (IPGeoFeed feed in geoFeeds)
{
csvContent.AppendLine($"{feed.Prefix},{feed.GeolocCountry},{feed.GeolocRegion},{feed.GeolocCity},{feed.GeolocPostalCode}");
}
return csvContent.ToString();
}
}
}

127
AS1024.GeoFeed/GeoFeedBuilder/NetBoxGeoFeedProvider.cs
View File

@@ -1,127 +0,0 @@
using AS1024.GeoFeed.Interfaces;
using AS1024.GeoFeed.Models;
using System.Text.Json;
using System.Net.Http.Headers;
using System.Net.Sockets;
using System.Web;
namespace AS1024.GeoFeed.GeoFeedBuilder
{
public class NetBoxGeoFeedProvider : IGeoFeedProvider
{
private readonly IConfiguration configuration;
private readonly ILogger<NetBoxGeoFeedProvider> logger;
private readonly IList<AddressFamily> addressFamilies = new List<AddressFamily>()
{
AddressFamily.InterNetwork,
AddressFamily.InterNetworkV6
};
private readonly IHttpClientFactory httpClientFactory;
string IGeoFeedProvider.GeoFeedProviderName => "netbox";
public NetBoxGeoFeedProvider(IConfiguration configuration,
ILogger<NetBoxGeoFeedProvider> logger,
IHttpClientFactory httpClientFactory)
{
this.configuration = configuration;
this.logger = logger;
this.httpClientFactory = httpClientFactory;
}
public async Task<List<IPGeoFeed>> GetGeoFeedData()
{
List<IPGeoFeed> geoFeed = [];
using HttpClient client = httpClientFactory.CreateClient();
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", configuration["APIKey"]);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
foreach (AddressFamily family in addressFamilies)
{
Uri uri = BuildNetBoxURI(family);
NetboxData? jsonData = null;
while (true)
{
logger.LogDebug($"Making request to {uri}...");
using HttpResponseMessage result = await client.GetAsync(uri);
if (!result.IsSuccessStatusCode)
{
break;
}
string stringResult = await result.Content.ReadAsStringAsync();
jsonData = JsonSerializer.Deserialize<NetboxData>(stringResult, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
});
if (jsonData?.Results == null || jsonData.Results.Count == 0)
{
break;
}
foreach (Result data in jsonData.Results)
{
try
{
geoFeed.Add(new IPGeoFeed
{
Prefix = data.Prefix,
GeolocCity = data.CustomFields.GeolocCity,
GeolocRegion = data.CustomFields.GeolocRegion,
GeolocCountry = data.CustomFields.GeolocCountry,
GeolocHasLocation = data.CustomFields.GeolocHasLocation,
GeolocPostalCode = data.CustomFields.GeolocPostalCode
});
}
catch (Exception ex)
{
logger.LogError(ex.ToString());
}
}
if (!string.IsNullOrEmpty(jsonData.Next))
{
uri = new Uri(jsonData.Next);
continue;
}
break;
}
}
return geoFeed;
}
protected Uri BuildNetBoxURI(AddressFamily family)
{
System.Collections.Specialized.NameValueCollection queryParameters = HttpUtility.ParseQueryString(string.Empty);
queryParameters["cf_geoloc_has_location"] = "true";
queryParameters["limit"] = "50";
switch (family)
{
case AddressFamily.InterNetwork:
queryParameters["mask_length__lte"] = "24";
queryParameters["family"] = "4";
break;
case AddressFamily.InterNetworkV6:
queryParameters["mask_length__lte"] = "48";
queryParameters["family"] = "6";
break;
}
UriBuilder endUrl = new UriBuilder
{
Path = "api/ipam/prefixes/",
Query = queryParameters.ToString(),
Host = configuration["NetBoxHost"],
Scheme = "https"
};
return endUrl.Uri;
}
}
}

51
AS1024.GeoFeed/GeoFeedBuilder/PreloadGeoFeed.cs
View File

@@ -1,51 +0,0 @@
using AS1024.GeoFeed.Interfaces;
using Microsoft.Extensions.Caching.Memory;
namespace AS1024.GeoFeed.GeoFeedBuilder
{
public class PreLoadGeoFeed : IHostedService
{
private readonly ILogger<PreLoadGeoFeed> logger;
private readonly IGeoFeedProvider provider;
private readonly IMemoryCache memoryCache;
private readonly IWebHostEnvironment environment;
private const string GeoFeedCacheKey = "GeoFeedData";
public PreLoadGeoFeed(ILogger<PreLoadGeoFeed> logger,
IGeoFeedProvider provider,
IMemoryCache memoryCache,
IWebHostEnvironment environment)
{
this.logger = logger;
this.provider = provider;
this.memoryCache = memoryCache;
this.environment = environment;
}
async Task IHostedService.StartAsync(CancellationToken cancellationToken)
{
try
{
if (environment.IsProduction())
await StartPreLoad();
} catch (Exception ex)
{
logger.LogWarning($"Failed to preload, exception settings below:\n{ex}");
}
}
private async Task StartPreLoad()
{
logger.LogInformation("Preloading GeoFeed data in memory...");
List<Models.IPGeoFeed> feed = await provider.GetGeoFeedData();
MemoryCacheEntryOptions cacheEntryOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(45));
memoryCache.Set(GeoFeedCacheKey, feed, cacheEntryOptions);
}
Task IHostedService.StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}

15
AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedCacheDbContext.cs
View File

@@ -1,15 +0,0 @@
using Microsoft.EntityFrameworkCore;
namespace AS1024.GeoFeed.GeoFeedLocalCache
{
public class GeoFeedCacheDbContext : DbContext
{
public GeoFeedCacheDbContext(DbContextOptions options)
: base(options)
{
}
public virtual DbSet<GeoFeedCacheEntry> GeoFeedCacheEntries { get; set; }
}
}

12
AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedCacheEntry.cs
View File

@@ -1,12 +0,0 @@
using AS1024.GeoFeed.Models;
using System.ComponentModel.DataAnnotations;
namespace AS1024.GeoFeed.GeoFeedLocalCache
{
public class GeoFeedCacheEntry : IPGeoFeed
{
[Key]
public int Id { get; set; }
}
}

90
AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedCacheService.cs
View File

@@ -1,90 +0,0 @@
using AS1024.GeoFeed.Interfaces;
using Microsoft.EntityFrameworkCore;
using System.Runtime.CompilerServices;
namespace AS1024.GeoFeed.GeoFeedLocalCache
{
public class GeoFeedCacheService : IHostedService
{
private readonly ILogger<GeoFeedCacheService> logger;
private readonly IGeoFeedProvider feedProvider;
private readonly IHost host;
public GeoFeedCacheService(ILogger<GeoFeedCacheService> logger,
IGeoFeedProvider feedProvider,
IHost host)
{
this.logger = logger;
this.feedProvider = feedProvider;
this.host = host;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_ = StartPerioidicSync(cancellationToken);
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public async Task<bool> StartPerioidicSync(CancellationToken Token)
{
await DBContextMigrate();
List<GeoFeedCacheEntry> geoFeedCacheEntry = [];
while (!Token.IsCancellationRequested)
{
logger.LogInformation("Running on disk fallback cache process...");
try
{
using var scope = host.Services.CreateScope();
using var dbContext = scope.ServiceProvider.GetRequiredService<GeoFeedCacheDbContext>();
var results = await feedProvider.GetGeoFeedData();
results.ForEach(x =>
{
geoFeedCacheEntry.Add(new()
{
Prefix = x.Prefix,
GeolocCity = x.GeolocCity,
GeolocCountry = x.GeolocCountry,
GeolocHasLocation = x.GeolocHasLocation,
GeolocPostalCode = x.GeolocPostalCode,
GeolocRegion = x.GeolocRegion
});
});
if (dbContext.GeoFeedCacheEntries.Any())
{
dbContext.GeoFeedCacheEntries.RemoveRange(dbContext.GeoFeedCacheEntries.ToArray());
}
await dbContext.AddRangeAsync(geoFeedCacheEntry, Token);
await dbContext.SaveChangesAsync(Token);
}
catch (Exception ex)
{
logger.LogWarning("On disk cache failed to run. Waiting on 30 minutes before retry...");
}
await Task.Delay(TimeSpan.FromMinutes(30));
}
return false;
}
private async Task DBContextMigrate()
{
using IServiceScope scope = host.Services.CreateScope();
using GeoFeedCacheDbContext? dbContext =
scope.ServiceProvider.GetService<GeoFeedCacheDbContext>();
#pragma warning disable CS8602 // Dereference of a possibly null reference.
if (dbContext.Database.GetPendingMigrations().Any()) {
await dbContext.Database.MigrateAsync();
}
#pragma warning restore CS8602 // Dereference of a possibly null reference.
}
}
}

16
AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedDesignTimeMigration.cs
View File

@@ -1,16 +0,0 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace AS1024.GeoFeed.GeoFeedLocalCache
{
public class GeoFeedDesignTimeMigration : IDesignTimeDbContextFactory<GeoFeedCacheDbContext>
{
public GeoFeedCacheDbContext CreateDbContext(string[] args)
{
var builder = new DbContextOptionsBuilder<GeoFeedCacheDbContext>();
builder.UseSqlite("Data Source=migratedb.db");
return new GeoFeedCacheDbContext(builder.Options);
}
}
}

10
AS1024.GeoFeed/Interfaces/GeoFeedProvider.cs
View File

@@ -1,10 +0,0 @@
using AS1024.GeoFeed.Models;
namespace AS1024.GeoFeed.Interfaces
{
public interface IGeoFeedProvider
{
public string GeoFeedProviderName { get; }
public Task<List<IPGeoFeed>> GetGeoFeedData();
}
}

54
AS1024.GeoFeed/Migrations/20240108180753_DiskCacheMigration.Designer.cs generated
View File

@@ -1,54 +0,0 @@
// <auto-generated />
using System;
using AS1024.GeoFeed.GeoFeedLocalCache;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace AS1024.GeoFeed.Migrations
{
[DbContext(typeof(GeoFeedCacheDbContext))]
[Migration("20240108180753_DiskCacheMigration")]
partial class DiskCacheMigration
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
modelBuilder.Entity("AS1024.GeoFeed.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("GeolocCity")
.HasColumnType("TEXT");
b.Property<string>("GeolocCountry")
.HasColumnType("TEXT");
b.Property<bool?>("GeolocHasLocation")
.HasColumnType("INTEGER");
b.Property<string>("GeolocPostalCode")
.HasColumnType("TEXT");
b.Property<string>("GeolocRegion")
.HasColumnType("TEXT");
b.Property<string>("Prefix")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("GeoFeedCacheEntries");
});
#pragma warning restore 612, 618
}
}
}

39
AS1024.GeoFeed/Migrations/20240108180753_DiskCacheMigration.cs
View File

@@ -1,39 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace AS1024.GeoFeed.Migrations
{
/// <inheritdoc />
public partial class DiskCacheMigration : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "GeoFeedCacheEntries",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
GeolocCity = table.Column<string>(type: "TEXT", nullable: true),
GeolocCountry = table.Column<string>(type: "TEXT", nullable: true),
GeolocHasLocation = table.Column<bool>(type: "INTEGER", nullable: true),
GeolocRegion = table.Column<string>(type: "TEXT", nullable: true),
GeolocPostalCode = table.Column<string>(type: "TEXT", nullable: true),
Prefix = table.Column<string>(type: "TEXT", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_GeoFeedCacheEntries", x => x.Id);
});
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "GeoFeedCacheEntries");
}
}
}

51
AS1024.GeoFeed/Migrations/GeoFeedCacheDbContextModelSnapshot.cs
View File

@@ -1,51 +0,0 @@
// <auto-generated />
using System;
using AS1024.GeoFeed.GeoFeedLocalCache;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace AS1024.GeoFeed.Migrations
{
[DbContext(typeof(GeoFeedCacheDbContext))]
partial class GeoFeedCacheDbContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
modelBuilder.Entity("AS1024.GeoFeed.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("GeolocCity")
.HasColumnType("TEXT");
b.Property<string>("GeolocCountry")
.HasColumnType("TEXT");
b.Property<bool?>("GeolocHasLocation")
.HasColumnType("INTEGER");
b.Property<string>("GeolocPostalCode")
.HasColumnType("TEXT");
b.Property<string>("GeolocRegion")
.HasColumnType("TEXT");
b.Property<string>("Prefix")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("GeoFeedCacheEntries");
});
#pragma warning restore 612, 618
}
}
}

50
AS1024.GeoFeed/Models/GeoFeed.cs
View File

@@ -1,50 +0,0 @@
using System.Text.Json.Serialization;
namespace AS1024.GeoFeed.Models
{
public class NetboxData
{
public List<Result>? Results { get; set; }
public string? Next { get; set; }
public string? Previous { get; set; }
}
public class Result
{
public string? Prefix { get; set; }
public CustomFields? CustomFields { get; set; }
}
public class CustomFields
{
/// <summary>
/// Represents the city associated with the IP address. This field is optional.
/// </summary>
public string? GeolocCity { get; set; }
/// <summary>
/// Represents the country associated with the IP address. This field is optional and expected to be a selection field in NetBox.
/// </summary>
public string? GeolocCountry { get; set; }
/// <summary>
/// Indicates whether geolocation data is available for the IP address. This field is required.
/// </summary>
public bool? GeolocHasLocation { get; set; }
/// <summary>
/// Represents the region or state associated with the IP address. This field is optional and expected to be a selection field in NetBox.
/// </summary>
public string? GeolocRegion { get; set; }
/// <summary>
/// Represents the postal code associated with the IP address. This field is optional.
/// </summary>
public string? GeolocPostalCode { get; set; }
}
/// <summary>
/// This class represents the IP GeoFeed Entry
/// </summary>
public class IPGeoFeed : CustomFields {
/// <summary>
/// Represents the IP Prefix for the associated GeoFeed entry
/// </summary>
public string? Prefix { get; set; }
}
}

7
AS1024.GeoFeed/Program.cs
View File

@@ -1,6 +1,7 @@
using AS1024.GeoFeed.GeoFeedBuilder;
using AS1024.GeoFeed.GeoFeedLocalCache;
using AS1024.GeoFeed.Interfaces;
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Core.GeoFeedPreloader;
using AS1024.GeoFeed.Core.GeoFeedLocalCache;
using AS1024.GeoFeed.Core.GeoFeedProviders;
using Microsoft.EntityFrameworkCore;
namespace AS1024.GeoFeed