AS1024/GeoFeed
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Inital work on breaking out the core logic out of the MVC API Web App in preparation to migrating to minimal API's in a seperate project

Browse Source
This commit is contained in:
Jeff Leung
2024-01-13 12:42:46 -08:00
parent 52708bd2c2
commit bdd247e6e0
18 changed files with 91 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
AS1024.GeoFeed.Core/AS1024.GeoFeed.Core.csproj Normal file
View File

@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\AS1024.GeoFeed.Models\AS1024.GeoFeed.Models.csproj" />
</ItemGroup>
</Project>

15
AS1024.GeoFeed.Core/GeoFeedLocalCache/GeoFeedCacheDbContext.cs Normal file
View File

@@ -0,0 +1,15 @@
using Microsoft.EntityFrameworkCore;
namespace AS1024.GeoFeed.Core.GeoFeedLocalCache
{
public class GeoFeedCacheDbContext : DbContext
{
public GeoFeedCacheDbContext(DbContextOptions options)
: base(options)
{
}
public virtual DbSet<GeoFeedCacheEntry> GeoFeedCacheEntries { get; set; }
}
}

12
AS1024.GeoFeed.Core/GeoFeedLocalCache/GeoFeedCacheEntry.cs Normal file
View File

@@ -0,0 +1,12 @@
using AS1024.GeoFeed.Models;
using System.ComponentModel.DataAnnotations;
namespace AS1024.GeoFeed.Core.GeoFeedLocalCache
{
public class GeoFeedCacheEntry : IPGeoFeed
{
[Key]
public int Id { get; set; }
}
}

93
AS1024.GeoFeed.Core/GeoFeedLocalCache/GeoFeedCacheService.cs Normal file
View File

@@ -0,0 +1,93 @@
using AS1024.GeoFeed.Core.Interfaces;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Runtime.CompilerServices;
namespace AS1024.GeoFeed.Core.GeoFeedLocalCache
{
public class GeoFeedCacheService : IHostedService
{
private readonly ILogger<GeoFeedCacheService> logger;
private readonly IGeoFeedProvider feedProvider;
private readonly IHost host;
public GeoFeedCacheService(ILogger<GeoFeedCacheService> logger,
IGeoFeedProvider feedProvider,
IHost host)
{
this.logger = logger;
this.feedProvider = feedProvider;
this.host = host;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_ = StartPerioidicSync(cancellationToken);
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public async Task<bool> StartPerioidicSync(CancellationToken Token)
{
await DBContextMigrate();
List<GeoFeedCacheEntry> geoFeedCacheEntry = [];
while (!Token.IsCancellationRequested)
{
logger.LogInformation("Running on disk fallback cache process...");
try
{
using var scope = host.Services.CreateScope();
using var dbContext = scope.ServiceProvider.GetRequiredService<GeoFeedCacheDbContext>();
var results = await feedProvider.GetGeoFeedData();
results.ForEach(x =>
{
geoFeedCacheEntry.Add(new()
{
Prefix = x.Prefix,
GeolocCity = x.GeolocCity,
GeolocCountry = x.GeolocCountry,
GeolocHasLocation = x.GeolocHasLocation,
GeolocPostalCode = x.GeolocPostalCode,
GeolocRegion = x.GeolocRegion
});
});
if (dbContext.GeoFeedCacheEntries.Any())
{
dbContext.GeoFeedCacheEntries.RemoveRange(dbContext.GeoFeedCacheEntries.ToArray());
}
await dbContext.AddRangeAsync(geoFeedCacheEntry, Token);
await dbContext.SaveChangesAsync(Token);
}
catch (Exception ex)
{
logger.LogWarning("On disk cache failed to run. Waiting on 30 minutes before retry...");
}
await Task.Delay(TimeSpan.FromMinutes(30));
}
return false;
}
private async Task DBContextMigrate()
{
using IServiceScope scope = host.Services.CreateScope();
using GeoFeedCacheDbContext? dbContext =
scope.ServiceProvider.GetService<GeoFeedCacheDbContext>();
#pragma warning disable CS8602 // Dereference of a possibly null reference.
if (dbContext.Database.GetPendingMigrations().Any()) {
await dbContext.Database.MigrateAsync();
}
#pragma warning restore CS8602 // Dereference of a possibly null reference.
}
}
}

16
AS1024.GeoFeed.Core/GeoFeedLocalCache/GeoFeedDesignTimeMigration.cs Normal file
View File

@@ -0,0 +1,16 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace AS1024.GeoFeed.Core.GeoFeedLocalCache
{
public class GeoFeedDesignTimeMigration : IDesignTimeDbContextFactory<GeoFeedCacheDbContext>
{
public GeoFeedCacheDbContext CreateDbContext(string[] args)
{
var builder = new DbContextOptionsBuilder<GeoFeedCacheDbContext>();
builder.UseSqlite("Data Source=migratedb.db");
return new GeoFeedCacheDbContext(builder.Options);
}
}
}

53
AS1024.GeoFeed.Core/GeoFeedPreloader/PreloadGeoFeed.cs Normal file
View File

@@ -0,0 +1,53 @@
using AS1024.GeoFeed.Core.Interfaces;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace AS1024.GeoFeed.Core.GeoFeedPreloader
{
public class PreLoadGeoFeed : IHostedService
{
private readonly ILogger<PreLoadGeoFeed> logger;
private readonly IGeoFeedProvider provider;
private readonly IMemoryCache memoryCache;
private readonly IHostEnvironment environment;
private const string GeoFeedCacheKey = "GeoFeedData";
public PreLoadGeoFeed(ILogger<PreLoadGeoFeed> logger,
IGeoFeedProvider provider,
IMemoryCache memoryCache,
IHostEnvironment environment)
{
this.logger = logger;
this.provider = provider;
this.memoryCache = memoryCache;
this.environment = environment;
}
async Task IHostedService.StartAsync(CancellationToken cancellationToken)
{
try
{
if (environment.IsProduction())
await StartPreLoad();
}
catch (Exception ex)
{
logger.LogWarning($"Failed to preload, exception settings below:\n{ex}");
}
}
private async Task StartPreLoad()
{
logger.LogInformation("Preloading GeoFeed data in memory...");
List<Models.IPGeoFeed> feed = await provider.GetGeoFeedData();
MemoryCacheEntryOptions cacheEntryOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(45));
memoryCache.Set(GeoFeedCacheKey, feed, cacheEntryOptions);
}
Task IHostedService.StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}

129
AS1024.GeoFeed.Core/GeoFeedProviders/NetBoxGeoFeedProvider.cs Normal file
View File

@@ -0,0 +1,129 @@
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Models;
using System.Text.Json;
using System.Net.Http.Headers;
using System.Net.Sockets;
using System.Web;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace AS1024.GeoFeed.Core.GeoFeedProviders
{
public class NetBoxGeoFeedProvider : IGeoFeedProvider
{
private readonly IConfiguration configuration;
private readonly ILogger<NetBoxGeoFeedProvider> logger;
private readonly IList<AddressFamily> addressFamilies = new List<AddressFamily>()
{
AddressFamily.InterNetwork,
AddressFamily.InterNetworkV6
};
private readonly IHttpClientFactory httpClientFactory;
string IGeoFeedProvider.GeoFeedProviderName => "netbox";
public NetBoxGeoFeedProvider(IConfiguration configuration,
ILogger<NetBoxGeoFeedProvider> logger,
IHttpClientFactory httpClientFactory)
{
this.configuration = configuration;
this.logger = logger;
this.httpClientFactory = httpClientFactory;
}
public async Task<List<IPGeoFeed>> GetGeoFeedData()
{
List<IPGeoFeed> geoFeed = [];
using HttpClient client = httpClientFactory.CreateClient();
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", configuration["APIKey"]);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
foreach (AddressFamily family in addressFamilies)
{
Uri uri = BuildNetBoxURI(family);
NetboxData? jsonData = null;
while (true)
{
logger.LogDebug($"Making request to {uri}...");
using HttpResponseMessage result = await client.GetAsync(uri);
if (!result.IsSuccessStatusCode)
{
break;
}
string stringResult = await result.Content.ReadAsStringAsync();
jsonData = JsonSerializer.Deserialize<NetboxData>(stringResult, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
});
if (jsonData?.Results == null || jsonData.Results.Count == 0)
{
break;
}
foreach (Result data in jsonData.Results)
{
try
{
geoFeed.Add(new IPGeoFeed
{
Prefix = data.Prefix,
GeolocCity = data.CustomFields.GeolocCity,
GeolocRegion = data.CustomFields.GeolocRegion,
GeolocCountry = data.CustomFields.GeolocCountry,
GeolocHasLocation = data.CustomFields.GeolocHasLocation,
GeolocPostalCode = data.CustomFields.GeolocPostalCode
});
}
catch (Exception ex)
{
logger.LogError(ex.ToString());
}
}
if (!string.IsNullOrEmpty(jsonData.Next))
{
uri = new Uri(jsonData.Next);
continue;
}
break;
}
}
return geoFeed;
}
protected Uri BuildNetBoxURI(AddressFamily family)
{
System.Collections.Specialized.NameValueCollection queryParameters = HttpUtility.ParseQueryString(string.Empty);
queryParameters["cf_geoloc_has_location"] = "true";
queryParameters["limit"] = "50";
switch (family)
{
case AddressFamily.InterNetwork:
queryParameters["mask_length__lte"] = "24";
queryParameters["family"] = "4";
break;
case AddressFamily.InterNetworkV6:
queryParameters["mask_length__lte"] = "48";
queryParameters["family"] = "6";
break;
}
UriBuilder endUrl = new UriBuilder
{
Path = "api/ipam/prefixes/",
Query = queryParameters.ToString(),
Host = configuration["NetBoxHost"],
Scheme = "https"
};
return endUrl.Uri;
}
}
}

10
AS1024.GeoFeed.Core/Interfaces/GeoFeedProvider.cs Normal file
View File

@@ -0,0 +1,10 @@
using AS1024.GeoFeed.Models;
namespace AS1024.GeoFeed.Core.Interfaces
{
public interface IGeoFeedProvider
{
public string GeoFeedProviderName { get; }
public Task<List<IPGeoFeed>> GetGeoFeedData();
}
}

54
AS1024.GeoFeed.Core/Migrations/20240113204143_InitialMigration.Designer.cs generated Normal file
View File

@@ -0,0 +1,54 @@
// <auto-generated />
using System;
using AS1024.GeoFeed.Core.GeoFeedLocalCache;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace AS1024.GeoFeed.Core.Migrations
{
[DbContext(typeof(GeoFeedCacheDbContext))]
[Migration("20240113204143_InitialMigration")]
partial class InitialMigration
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
modelBuilder.Entity("AS1024.GeoFeed.Core.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("GeolocCity")
.HasColumnType("TEXT");
b.Property<string>("GeolocCountry")
.HasColumnType("TEXT");
b.Property<bool?>("GeolocHasLocation")
.HasColumnType("INTEGER");
b.Property<string>("GeolocPostalCode")
.HasColumnType("TEXT");
b.Property<string>("GeolocRegion")
.HasColumnType("TEXT");
b.Property<string>("Prefix")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("GeoFeedCacheEntries");
});
#pragma warning restore 612, 618
}
}
}

39
AS1024.GeoFeed.Core/Migrations/20240113204143_InitialMigration.cs Normal file
View File

@@ -0,0 +1,39 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace AS1024.GeoFeed.Core.Migrations
{
/// <inheritdoc />
public partial class InitialMigration : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "GeoFeedCacheEntries",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
GeolocCity = table.Column<string>(type: "TEXT", nullable: true),
GeolocCountry = table.Column<string>(type: "TEXT", nullable: true),
GeolocHasLocation = table.Column<bool>(type: "INTEGER", nullable: true),
GeolocRegion = table.Column<string>(type: "TEXT", nullable: true),
GeolocPostalCode = table.Column<string>(type: "TEXT", nullable: true),
Prefix = table.Column<string>(type: "TEXT", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_GeoFeedCacheEntries", x => x.Id);
});
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "GeoFeedCacheEntries");
}
}
}

51
AS1024.GeoFeed.Core/Migrations/GeoFeedCacheDbContextModelSnapshot.cs Normal file
View File

@@ -0,0 +1,51 @@
// <auto-generated />
using System;
using AS1024.GeoFeed.Core.GeoFeedLocalCache;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace AS1024.GeoFeed.Core.Migrations
{
[DbContext(typeof(GeoFeedCacheDbContext))]
partial class GeoFeedCacheDbContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
modelBuilder.Entity("AS1024.GeoFeed.Core.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("GeolocCity")
.HasColumnType("TEXT");
b.Property<string>("GeolocCountry")
.HasColumnType("TEXT");
b.Property<bool?>("GeolocHasLocation")
.HasColumnType("INTEGER");
b.Property<string>("GeolocPostalCode")
.HasColumnType("TEXT");
b.Property<string>("GeolocRegion")
.HasColumnType("TEXT");
b.Property<string>("Prefix")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("GeoFeedCacheEntries");
});
#pragma warning restore 612, 618
}
}
}

20
AS1024.GeoFeed.Core/Tools/GeoFeedTools.cs Normal file
View File

@@ -0,0 +1,20 @@
using AS1024.GeoFeed.Models;
using System.Text;
namespace AS1024.GeoFeed.Core.Tools
{
public static class GeoFeedTools
{
public static string ToGeoFeedCsv(this List<IPGeoFeed> geoFeeds)
{
StringBuilder csvContent = new();
foreach (IPGeoFeed feed in geoFeeds)
{
csvContent.AppendLine($"{feed.Prefix},{feed.GeolocCountry},{feed.GeolocRegion},{feed.GeolocCity},{feed.GeolocPostalCode}");
}
return csvContent.ToString();
}
}
}