Compare commits

...

19 Commits

Author SHA1 Message Date
Jeff Leung 690a117ffd Update README 2024-01-14 00:08:14 -08:00
Jeff Leung 9dfbded5b8 Update the readme file 2024-01-14 00:04:14 -08:00
Jeff Leung 7b7d422890 Update the README.md file 2024-01-13 23:56:06 -08:00
Jeff Leung 2d6083f530 Don't add unecessary binaries 2024-01-13 23:38:18 -08:00
Jeff Leung 0e56d778bc Mark class as abstract so we don't instantiate or inject it by accident 2024-01-13 19:43:29 -08:00
Jeff Leung 8523ddd60f Add Alpine deployment 2024-01-13 19:41:34 -08:00
Jeff Leung 83826cc930 Switch over to jammy 2024-01-13 19:28:03 -08:00
Jeff Leung cc1717e4a3 Return a file instead of just a CSV text 2024-01-13 18:11:07 -08:00
Jeff Leung 48932ec2a5 Don't include the symbols 2024-01-13 17:44:44 -08:00
Jeff Leung 42aacf497c Some minor housekeeping 2024-01-13 17:10:18 -08:00
Jeff Leung e8152186c1 Bring the caching services over 2024-01-13 17:05:21 -08:00
Jeff Leung 43b34143a3 Remove uneeded usings 2024-01-13 17:05:08 -08:00
Jeff Leung 8ed021754c Remove uneeded files 2024-01-13 17:04:55 -08:00
Jeff Leung a44b4f9421 Initial work on getting minimal API to work 2024-01-13 16:44:03 -08:00
Jeff Leung 810993cbf3 Update launch settings 2024-01-13 16:43:44 -08:00
Jeff Leung 8d6999c95c Move it to a base class so we can override the deserializer in a different project 2024-01-13 14:20:38 -08:00
Jeff Leung 704d6b24dc Don't reference to EF Core as AOT does not yet support it 2024-01-13 13:44:20 -08:00
Jeff Leung 81d01f28a3 Add initial work to get minimal API's going 2024-01-13 13:30:28 -08:00
Jeff Leung bdd247e6e0 Inital work on breaking out the core logic out of the MVC API Web App in preparation to migrating to minimal API's in a seperate project 2024-01-13 12:42:46 -08:00
34 changed files with 713 additions and 146 deletions

View File

@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\AS1024.GeoFeed.Core\AS1024.GeoFeed.Core.csproj" />
<ProjectReference Include="..\AS1024.GeoFeed.Models\AS1024.GeoFeed.Models.csproj" />
</ItemGroup>
</Project>

View File

@ -1,6 +1,6 @@
using Microsoft.EntityFrameworkCore;
namespace AS1024.GeoFeed.GeoFeedLocalCache
namespace AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache
{
public class GeoFeedCacheDbContext : DbContext
{

View File

@ -1,7 +1,7 @@
using AS1024.GeoFeed.Models;
using System.ComponentModel.DataAnnotations;
namespace AS1024.GeoFeed.GeoFeedLocalCache
namespace AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache
{
public class GeoFeedCacheEntry : IPGeoFeed
{

View File

@ -1,7 +1,8 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace AS1024.GeoFeed.GeoFeedLocalCache
namespace AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache
{
public class GeoFeedDesignTimeMigration : IDesignTimeDbContextFactory<GeoFeedCacheDbContext>
{

View File

@ -0,0 +1,72 @@
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Hosting;
using AS1024.GeoFeed.Core.Tools;
namespace AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache
{
public class GeoFeedSqliteCache : IGeoFeedPersistentCacheProvider
{
protected readonly GeoFeedCacheDbContext dbContext;
private readonly IGeoFeedProvider feedProvider;
public GeoFeedSqliteCache(GeoFeedCacheDbContext geoFeedCacheDb,
IHost host,
IGeoFeedProvider provider)
{
dbContext = geoFeedCacheDb;
feedProvider = provider;
}
public string ProviderName => "sqlite";
public async Task<bool> CacheGeoFeed(IList<IPGeoFeed> pGeoFeeds)
{
await DBContextMigrate();
List<GeoFeedCacheEntry> geoFeedCacheEntry = [];
var results = pGeoFeeds.ToList();
results.ForEach(x =>
{
geoFeedCacheEntry.Add(new()
{
Prefix = x.Prefix,
GeolocCity = x.GeolocCity,
GeolocCountry = x.GeolocCountry,
GeolocHasLocation = x.GeolocHasLocation,
GeolocPostalCode = x.GeolocPostalCode,
GeolocRegion = x.GeolocRegion
});
});
if (dbContext.GeoFeedCacheEntries.Any())
{
dbContext.GeoFeedCacheEntries.RemoveRange(dbContext.GeoFeedCacheEntries.ToArray());
}
await dbContext.AddRangeAsync(geoFeedCacheEntry);
await dbContext.SaveChangesAsync();
return true;
}
public string GetGeoFeed()
{
var results =
dbContext.GeoFeedCacheEntries.ToList();
List<IPGeoFeed> cachedData = [];
results.ForEach(cachedData.Add);
return cachedData.ToGeoFeedCsv();
}
private async Task DBContextMigrate()
{
if (dbContext.Database.GetPendingMigrations().Any())
{
await dbContext.Database.MigrateAsync();
}
}
}
}

View File

@ -1,6 +1,6 @@
// <auto-generated />
using System;
using AS1024.GeoFeed.GeoFeedLocalCache;
using AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@ -8,11 +8,11 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace AS1024.GeoFeed.Migrations
namespace AS1024.GeoFeed.Core.SqliteGeoFeedCache.Migrations
{
[DbContext(typeof(GeoFeedCacheDbContext))]
[Migration("20240108180753_DiskCacheMigration")]
partial class DiskCacheMigration
[Migration("20240114002908_InitialMigration")]
partial class InitialMigration
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
@ -20,7 +20,7 @@ namespace AS1024.GeoFeed.Migrations
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
modelBuilder.Entity("AS1024.GeoFeed.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
modelBuilder.Entity("AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache.GeoFeedCacheEntry", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()

View File

@ -2,10 +2,10 @@
#nullable disable
namespace AS1024.GeoFeed.Migrations
namespace AS1024.GeoFeed.Core.SqliteGeoFeedCache.Migrations
{
/// <inheritdoc />
public partial class DiskCacheMigration : Migration
public partial class InitialMigration : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)

View File

@ -1,13 +1,13 @@
// <auto-generated />
using System;
using AS1024.GeoFeed.GeoFeedLocalCache;
using AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace AS1024.GeoFeed.Migrations
namespace AS1024.GeoFeed.Core.SqliteGeoFeedCache.Migrations
{
[DbContext(typeof(GeoFeedCacheDbContext))]
partial class GeoFeedCacheDbContextModelSnapshot : ModelSnapshot
@ -17,7 +17,7 @@ namespace AS1024.GeoFeed.Migrations
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
modelBuilder.Entity("AS1024.GeoFeed.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
modelBuilder.Entity("AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache.GeoFeedCacheEntry", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()

View File

@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\AS1024.GeoFeed.Models\AS1024.GeoFeed.Models.csproj" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,58 @@
using AS1024.GeoFeed.Core.Interfaces;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace AS1024.GeoFeed.Core.CacheService
{
public class GeoFeedCacheService : IHostedService
{
private readonly ILogger<GeoFeedCacheService> logger;
private readonly IGeoFeedProvider feedProvider;
private readonly IHost host;
public GeoFeedCacheService(ILogger<GeoFeedCacheService> logger,
IGeoFeedProvider feedProvider,
IHost host)
{
this.logger = logger;
this.feedProvider = feedProvider;
this.host = host;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_ = StartPerioidicSync(cancellationToken);
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public async Task<bool> StartPerioidicSync(CancellationToken Token)
{
while (!Token.IsCancellationRequested)
{
try
{
var scope = host.Services.CreateScope();
var persistentCacheProvider =
scope.ServiceProvider.GetRequiredService<IGeoFeedPersistentCacheProvider>();
var results = await feedProvider.GetGeoFeedData();
await persistentCacheProvider.CacheGeoFeed(results);
}
catch (Exception)
{
logger.LogWarning("On disk cache failed to run. Waiting on 30 minutes before retry...");
}
await Task.Delay(TimeSpan.FromMinutes(30));
}
return false;
}
}
}

View File

@ -0,0 +1,55 @@
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Models;
using AS1024.GeoFeed.Core.Tools;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace AS1024.GeoFeed.Core.CacheService
{
public class GeoFeedLocalFileCache : IGeoFeedPersistentCacheProvider
{
private readonly IConfiguration configuration;
private readonly ILogger<GeoFeedLocalFileCache> logger;
public GeoFeedLocalFileCache(IConfiguration _configuration,
ILogger<GeoFeedLocalFileCache> logger)
{
configuration = _configuration;
this.logger = logger;
}
string IGeoFeedPersistentCacheProvider.ProviderName => "file";
Task<bool> IGeoFeedPersistentCacheProvider.CacheGeoFeed(IList<IPGeoFeed> pGeoFeeds)
{
string? tempPath = GetTempPath();
logger.LogInformation($"Writing geofeed data to: {tempPath}");
File.WriteAllText(tempPath, pGeoFeeds.ToList().
ToGeoFeedCsv());
return Task.FromResult(true);
}
private string GetTempPath()
{
string? tempPath = Path.Combine(Path.GetTempPath(), "geoFeedCache.csv");
logger.LogInformation($"Getting GeoFeed data from: {tempPath}");
if (!string.IsNullOrEmpty(configuration["TempCache"]))
tempPath = configuration["TempCache"];
return tempPath;
}
string IGeoFeedPersistentCacheProvider.GetGeoFeed()
{
return File.ReadAllText(GetTempPath());
}
}
}

View File

@ -1,21 +1,23 @@

using AS1024.GeoFeed.Interfaces;
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Models;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace AS1024.GeoFeed.GeoFeedBuilder
namespace AS1024.GeoFeed.Core.GeoFeedPreloader
{
public class PreLoadGeoFeed : IHostedService
{
private readonly ILogger<PreLoadGeoFeed> logger;
private readonly IGeoFeedProvider provider;
private readonly IMemoryCache memoryCache;
private readonly IWebHostEnvironment environment;
private readonly IHostEnvironment environment;
private const string GeoFeedCacheKey = "GeoFeedData";
public PreLoadGeoFeed(ILogger<PreLoadGeoFeed> logger,
IGeoFeedProvider provider,
IMemoryCache memoryCache,
IWebHostEnvironment environment)
IHostEnvironment environment)
{
this.logger = logger;
this.provider = provider;
@ -29,7 +31,8 @@ namespace AS1024.GeoFeed.GeoFeedBuilder
{
if (environment.IsProduction())
await StartPreLoad();
} catch (Exception ex)
}
catch (Exception ex)
{
logger.LogWarning($"Failed to preload, exception settings below:\n{ex}");
}
@ -38,7 +41,7 @@ namespace AS1024.GeoFeed.GeoFeedBuilder
private async Task StartPreLoad()
{
logger.LogInformation("Preloading GeoFeed data in memory...");
List<Models.IPGeoFeed> feed = await provider.GetGeoFeedData();
List<IPGeoFeed> feed = await provider.GetGeoFeedData();
MemoryCacheEntryOptions cacheEntryOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(45));
memoryCache.Set(GeoFeedCacheKey, feed, cacheEntryOptions);
}

View File

@ -0,0 +1,14 @@
using AS1024.GeoFeed.Core.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace AS1024.GeoFeed.Core.GeoFeedProviders
{
public class NetBoxGeoFeedProvider : NetBoxGeoFeedProviderBase, IGeoFeedProvider
{
public NetBoxGeoFeedProvider(IConfiguration configuration, ILogger<NetBoxGeoFeedProvider> logger, IHttpClientFactory httpClientFactory)
: base(configuration, logger, httpClientFactory)
{
}
}
}

View File

@ -1,26 +1,28 @@
using AS1024.GeoFeed.Interfaces;
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Models;
using System.Text.Json;
using System.Net.Http.Headers;
using System.Net.Sockets;
using System.Web;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace AS1024.GeoFeed.GeoFeedBuilder
namespace AS1024.GeoFeed.Core.GeoFeedProviders
{
public class NetBoxGeoFeedProvider : IGeoFeedProvider
public abstract class NetBoxGeoFeedProviderBase : IGeoFeedProvider
{
private readonly IConfiguration configuration;
private readonly ILogger<NetBoxGeoFeedProvider> logger;
private readonly IList<AddressFamily> addressFamilies = new List<AddressFamily>()
protected readonly IConfiguration configuration;
protected readonly ILogger<NetBoxGeoFeedProvider> logger;
protected readonly IList<AddressFamily> addressFamilies = new List<AddressFamily>()
{
AddressFamily.InterNetwork,
AddressFamily.InterNetworkV6
};
private readonly IHttpClientFactory httpClientFactory;
protected readonly IHttpClientFactory httpClientFactory;
string IGeoFeedProvider.GeoFeedProviderName => "netbox";
public NetBoxGeoFeedProvider(IConfiguration configuration,
public NetBoxGeoFeedProviderBase(IConfiguration configuration,
ILogger<NetBoxGeoFeedProvider> logger,
IHttpClientFactory httpClientFactory)
{
@ -51,10 +53,7 @@ namespace AS1024.GeoFeed.GeoFeedBuilder
break;
}
string stringResult = await result.Content.ReadAsStringAsync();
jsonData = JsonSerializer.Deserialize<NetboxData>(stringResult, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
});
jsonData = DeserializeJsonData(stringResult);
if (jsonData?.Results == null || jsonData.Results.Count == 0)
{
@ -94,6 +93,14 @@ namespace AS1024.GeoFeed.GeoFeedBuilder
return geoFeed;
}
protected virtual NetboxData? DeserializeJsonData(string stringResult)
{
return JsonSerializer.Deserialize<NetboxData>(stringResult, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
});
}
protected Uri BuildNetBoxURI(AddressFamily family)
{
System.Collections.Specialized.NameValueCollection queryParameters = HttpUtility.ParseQueryString(string.Empty);

View File

@ -0,0 +1,31 @@
using AS1024.GeoFeed.Models;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace AS1024.GeoFeed.Core.Interfaces
{
/// <summary>
/// Represents a persistent cache GeoFeed provider
/// </summary>
public interface IGeoFeedPersistentCacheProvider
{
/// <summary>
/// Name of the provider
/// </summary>
public string ProviderName { get; }
/// <summary>
/// Returns the GeoFeed
/// </summary>
/// <returns>String of the CSV geofeed</returns>
public string GetGeoFeed();
/// <summary>
/// Stores the GeoFeed in the cache backend
/// </summary>
/// <param name="pGeoFeeds">GeoFeed retrieved from the backend</param>
/// <returns></returns>
public Task<bool> CacheGeoFeed(IList<IPGeoFeed> pGeoFeeds);
}
}

View File

@ -1,6 +1,6 @@
using AS1024.GeoFeed.Models;
namespace AS1024.GeoFeed.Interfaces
namespace AS1024.GeoFeed.Core.Interfaces
{
public interface IGeoFeedProvider
{

View File

@ -1,7 +1,7 @@
using AS1024.GeoFeed.Models;
using System.Text;
namespace AS1024.GeoFeed.GeoFeedBuilder
namespace AS1024.GeoFeed.Core.Tools
{
public static class GeoFeedTools
{

View File

@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
<PublishAot>true</PublishAot>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\AS1024.GeoFeed.Core\AS1024.GeoFeed.Core.csproj" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,15 @@
using AS1024.GeoFeed.Models;
using System.Text.Json.Serialization;
namespace AS1024.GeoFeed.MinimalAPI
{
[JsonSerializable(typeof(NetboxData))]
[JsonSerializable(typeof(Result))]
[JsonSerializable(typeof(CustomFields))]
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower)]
internal partial class AppJsonSerializerContext : JsonSerializerContext
{
}
}

View File

@ -0,0 +1,29 @@
#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
FROM mcr.microsoft.com/dotnet/runtime:8.0-jammy-chiseled AS base
USER app
WORKDIR /app
EXPOSE 8080
FROM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build
# Install clang/zlib1g-dev dependencies for publishing to native
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
clang zlib1g-dev
ARG BUILD_CONFIGURATION=Release
WORKDIR /src
COPY ["AS1024.GeoFeed.MinimalAPI/AS1024.GeoFeed.MinimalAPI.csproj", "AS1024.GeoFeed.MinimalAPI/"]
RUN dotnet restore "./AS1024.GeoFeed.MinimalAPI/./AS1024.GeoFeed.MinimalAPI.csproj"
COPY . .
WORKDIR "/src/AS1024.GeoFeed.MinimalAPI"
RUN dotnet build "./AS1024.GeoFeed.MinimalAPI.csproj" -c $BUILD_CONFIGURATION -o /app/build
FROM build AS publish
ARG BUILD_CONFIGURATION=Release
RUN dotnet publish "./AS1024.GeoFeed.MinimalAPI.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=true /p:DebugType=None /p:DebugSymbols=false
FROM base AS final
WORKDIR /app
EXPOSE 8080
COPY --from=publish /app/publish .
ENTRYPOINT ["./AS1024.GeoFeed.MinimalAPI"]

View File

@ -0,0 +1,55 @@
#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
FROM alpine:latest AS base
ENV \
# UID of the non-root user 'app'
APP_UID=1654 \
# Configure web servers to bind to port 8080 when present
ASPNETCORE_HTTP_PORTS=8080 \
# Enable detection of running in a container
DOTNET_RUNNING_IN_CONTAINER=true \
# Set the invariant mode since ICU package isn't included (see https://github.com/dotnet/announcements/issues/20)
DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true
#RUN apk add --upgrade --no-cache \
# ca-certificates-bundle \
# \
# # .NET dependencies
# libgcc \
# libssl3 \
# libstdc++ \
# zlib
# Create a non-root user and group
RUN addgroup \
--gid=$APP_UID \
app \
&& adduser \
--uid=$APP_UID \
--ingroup=app \
--disabled-password \
app
WORKDIR /app
USER app
EXPOSE 8080
FROM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
# Install clang/zlib1g-dev dependencies for publishing to native
RUN apk add clang zlib-dev musl-dev libc6-compat
ARG BUILD_CONFIGURATION=Release
WORKDIR /src
COPY ["AS1024.GeoFeed.MinimalAPI/AS1024.GeoFeed.MinimalAPI.csproj", "AS1024.GeoFeed.MinimalAPI/"]
RUN dotnet restore "./AS1024.GeoFeed.MinimalAPI/./AS1024.GeoFeed.MinimalAPI.csproj"
COPY . .
WORKDIR "/src/AS1024.GeoFeed.MinimalAPI"
RUN dotnet build "./AS1024.GeoFeed.MinimalAPI.csproj" -c $BUILD_CONFIGURATION -o /app/build
FROM build AS publish
ARG BUILD_CONFIGURATION=Release
RUN dotnet publish "./AS1024.GeoFeed.MinimalAPI.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=true /p:DebugType=None /p:DebugSymbols=false
FROM base AS final
WORKDIR /app
EXPOSE 8080
COPY --from=publish /app/publish .
ENTRYPOINT ["./AS1024.GeoFeed.MinimalAPI"]

View File

@ -0,0 +1,22 @@
using AS1024.GeoFeed.Core.GeoFeedProviders;
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Models;
using System.Text.Json;
namespace AS1024.GeoFeed.MinimalAPI
{
internal class NetboxAoTGeoFeedProvider : NetBoxGeoFeedProvider, IGeoFeedProvider
{
public NetboxAoTGeoFeedProvider(IConfiguration configuration,
ILogger<NetBoxGeoFeedProvider> logger,
IHttpClientFactory httpClientFactory)
: base(configuration, logger, httpClientFactory)
{
}
protected override NetboxData? DeserializeJsonData(string stringResult)
{
return JsonSerializer.Deserialize(stringResult, AppJsonSerializerContext.Default.NetboxData);
}
}
}

View File

@ -0,0 +1,93 @@
using AS1024.GeoFeed.Core.CacheService;
using AS1024.GeoFeed.Core.GeoFeedProviders;
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Core.Tools;
using AS1024.GeoFeed.Models;
using Microsoft.AspNetCore.Http.HttpResults;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Text;
namespace AS1024.GeoFeed.MinimalAPI
{
public class Program
{
public static void Main(string[] args)
{
var builder = WebApplication.CreateSlimBuilder(args);
builder.Services.AddTransient<IGeoFeedProvider, NetboxAoTGeoFeedProvider>();
builder.Services.AddHostedService<GeoFeedCacheService>();
builder.Services.AddTransient<IGeoFeedPersistentCacheProvider, GeoFeedLocalFileCache>();
builder.Services.AddMemoryCache();
builder.Services.AddLogging();
builder.Services.AddHttpClient();
builder.Services.ConfigureHttpJsonOptions(options => {
options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default);
});
var app = builder.Build();
app.Map("/geofeed.csv", async (IGeoFeedProvider provider,
ILogger<Program> logger,
IGeoFeedPersistentCacheProvider cacheProvider,
IMemoryCache memoryCache,
IWebHostEnvironment environment) => {
return await GeoFeedDataRunner(provider, logger, cacheProvider, memoryCache, environment);
});
app.Map("/geofeed", async (IGeoFeedProvider provider,
ILogger<Program> logger,
IGeoFeedPersistentCacheProvider cacheProvider,
IMemoryCache memoryCache,
IWebHostEnvironment environment) => {
return await GeoFeedDataRunner(provider, logger, cacheProvider, memoryCache, environment);
});
app.Run();
}
protected static async Task<IResult> GeoFeedDataRunner(IGeoFeedProvider provider,
ILogger<Program> logger,
IGeoFeedPersistentCacheProvider cacheProvider,
IMemoryCache memoryCache,
IWebHostEnvironment environment)
{
try
{
if (!memoryCache.TryGetValue("Geofeed", out List<IPGeoFeed>? feed))
{
feed = await provider.GetGeoFeedData();
if (environment.IsProduction())
{
MemoryCacheEntryOptions cacheEntryOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(TimeSpan.FromMinutes(15));
memoryCache.Set("Geofeed", feed, cacheEntryOptions);
}
}
return Results.File(Encoding.UTF8.GetBytes(feed.ToGeoFeedCsv()),
"text/csv",
"geofeed.csv");
}
catch (HttpRequestException ex)
{
logger.LogWarning($"Temporary failure of retrieving GeoData from upstream. {ex}");
string geoFeedData = cacheProvider.GetGeoFeed();
return Results.File(Encoding.UTF8.GetBytes(geoFeedData),
"text/csv",
"geofeed.csv");
}
catch (Exception ex)
{
logger.LogError($"Error: {ex}");
}
return Results.NoContent();
}
}
}

View File

@ -0,0 +1,24 @@
{
"profiles": {
"http": {
"commandName": "Project",
"launchBrowser": true,
"launchUrl": "geofeed.csv",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"dotnetRunMessages": true,
"applicationUrl": "http://localhost:5127"
},
"Docker": {
"commandName": "Docker",
"launchBrowser": true,
"launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/todos",
"environmentVariables": {
"ASPNETCORE_HTTP_PORTS": "8080"
},
"publishAllPorts": true
}
},
"$schema": "http://json.schemastore.org/launchsettings.json"
}

View File

@ -0,0 +1,8 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
}
}

View File

@ -0,0 +1,9 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*"
}

View File

@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View File

@ -3,7 +3,18 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.8.34330.188
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AS1024.GeoFeed", "AS1024.GeoFeed\AS1024.GeoFeed.csproj", "{6292097C-7F35-45BB-B2B0-1918DF49FE7D}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AS1024.GeoFeed", "AS1024.GeoFeed\AS1024.GeoFeed.csproj", "{6292097C-7F35-45BB-B2B0-1918DF49FE7D}"
ProjectSection(ProjectDependencies) = postProject
{8BA82A53-29E7-44A2-91BA-57C15BB4B0F5} = {8BA82A53-29E7-44A2-91BA-57C15BB4B0F5}
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AS1024.GeoFeed.Models", "AS1024.GeoFeed.Models\AS1024.GeoFeed.Models.csproj", "{8BA82A53-29E7-44A2-91BA-57C15BB4B0F5}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AS1024.GeoFeed.Core", "AS1024.GeoFeed.Core\AS1024.GeoFeed.Core.csproj", "{EE6D6C87-29C4-42A1-8251-7D2BF20F1797}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AS1024.GeoFeed.MinimalAPI", "AS1024.GeoFeed.MinimalAPI\AS1024.GeoFeed.MinimalAPI.csproj", "{36F2958C-8D0E-463B-9BF3-D6E55E6FC0B8}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AS1024.GeoFeed.Core.SqliteGeoFeedCache", "AS1024.GeoFeed.Core.SqliteGeoFeedCache\AS1024.GeoFeed.Core.SqliteGeoFeedCache.csproj", "{3459BB31-FA7A-44D1-872D-C5338ACFBF80}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -15,6 +26,22 @@ Global
{6292097C-7F35-45BB-B2B0-1918DF49FE7D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6292097C-7F35-45BB-B2B0-1918DF49FE7D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6292097C-7F35-45BB-B2B0-1918DF49FE7D}.Release|Any CPU.Build.0 = Release|Any CPU
{8BA82A53-29E7-44A2-91BA-57C15BB4B0F5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8BA82A53-29E7-44A2-91BA-57C15BB4B0F5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8BA82A53-29E7-44A2-91BA-57C15BB4B0F5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8BA82A53-29E7-44A2-91BA-57C15BB4B0F5}.Release|Any CPU.Build.0 = Release|Any CPU
{EE6D6C87-29C4-42A1-8251-7D2BF20F1797}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EE6D6C87-29C4-42A1-8251-7D2BF20F1797}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EE6D6C87-29C4-42A1-8251-7D2BF20F1797}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EE6D6C87-29C4-42A1-8251-7D2BF20F1797}.Release|Any CPU.Build.0 = Release|Any CPU
{36F2958C-8D0E-463B-9BF3-D6E55E6FC0B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{36F2958C-8D0E-463B-9BF3-D6E55E6FC0B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{36F2958C-8D0E-463B-9BF3-D6E55E6FC0B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{36F2958C-8D0E-463B-9BF3-D6E55E6FC0B8}.Release|Any CPU.Build.0 = Release|Any CPU
{3459BB31-FA7A-44D1-872D-C5338ACFBF80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3459BB31-FA7A-44D1-872D-C5338ACFBF80}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3459BB31-FA7A-44D1-872D-C5338ACFBF80}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3459BB31-FA7A-44D1-872D-C5338ACFBF80}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

View File

@ -21,4 +21,10 @@
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\AS1024.GeoFeed.Core.SqliteGeoFeedCache\AS1024.GeoFeed.Core.SqliteGeoFeedCache.csproj" />
<ProjectReference Include="..\AS1024.GeoFeed.Core\AS1024.GeoFeed.Core.csproj" />
<ProjectReference Include="..\AS1024.GeoFeed.Models\AS1024.GeoFeed.Models.csproj" />
</ItemGroup>
</Project>

View File

@ -1,10 +1,9 @@
using Microsoft.AspNetCore.Mvc;
using AS1024.GeoFeed.Interfaces;
using AS1024.GeoFeed.GeoFeedBuilder;
using AS1024.GeoFeed.Core.Interfaces;
using Microsoft.Extensions.Caching.Memory;
using AS1024.GeoFeed.Models;
using System.Text;
using AS1024.GeoFeed.GeoFeedLocalCache;
using AS1024.GeoFeed.Core.Tools;
namespace AS1024.GeoFeed.Controllers
{
@ -18,16 +17,16 @@ namespace AS1024.GeoFeed.Controllers
private readonly IMemoryCache memoryCache;
private readonly IWebHostEnvironment environment;
private readonly ILogger<GeofeedController> logger;
private readonly GeoFeedCacheDbContext dbContext;
private readonly IGeoFeedPersistentCacheProvider geoFeedPersistentCache;
private const string GeoFeedCacheKey = "GeoFeedData";
public GeofeedController(IGeoFeedProvider builder,
IMemoryCache memoryCache,
IWebHostEnvironment environment,
ILogger<GeofeedController> logger,
GeoFeedCacheDbContext dbContext) {
IGeoFeedPersistentCacheProvider geoFeedPersistentCache) {
this.logger = logger;
this.dbContext = dbContext;
this.geoFeedPersistentCache = geoFeedPersistentCache;
this.builder = builder;
this.memoryCache = memoryCache;
this.environment = environment;
@ -54,10 +53,7 @@ namespace AS1024.GeoFeed.Controllers
} catch (HttpRequestException ex)
{
logger.LogWarning($"Temporary failure of retrieving GeoData from upstream. {ex}");
var results =
dbContext.GeoFeedCacheEntries.ToList();
List<IPGeoFeed> cachedData = [];
results.ForEach(cachedData.Add);
var cachedData = geoFeedPersistentCache.GetGeoFeed();
return ReturnFile(cachedData);
}
@ -73,6 +69,12 @@ namespace AS1024.GeoFeed.Controllers
private IActionResult ReturnFile(List<IPGeoFeed>? feed)
{
string csvContent = feed.ToGeoFeedCsv(); // Assuming ToGeoFeedCsv() returns a string in CSV format.
return ReturnFile(csvContent);
}
[NonAction]
private IActionResult ReturnFile(string csvContent)
{
byte[] contentBytes = Encoding.UTF8.GetBytes(csvContent);
string contentType = "text/csv";

View File

@ -1,90 +0,0 @@
using AS1024.GeoFeed.Interfaces;
using Microsoft.EntityFrameworkCore;
using System.Runtime.CompilerServices;
namespace AS1024.GeoFeed.GeoFeedLocalCache
{
public class GeoFeedCacheService : IHostedService
{
private readonly ILogger<GeoFeedCacheService> logger;
private readonly IGeoFeedProvider feedProvider;
private readonly IHost host;
public GeoFeedCacheService(ILogger<GeoFeedCacheService> logger,
IGeoFeedProvider feedProvider,
IHost host)
{
this.logger = logger;
this.feedProvider = feedProvider;
this.host = host;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_ = StartPerioidicSync(cancellationToken);
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public async Task<bool> StartPerioidicSync(CancellationToken Token)
{
await DBContextMigrate();
List<GeoFeedCacheEntry> geoFeedCacheEntry = [];
while (!Token.IsCancellationRequested)
{
logger.LogInformation("Running on disk fallback cache process...");
try
{
using var scope = host.Services.CreateScope();
using var dbContext = scope.ServiceProvider.GetRequiredService<GeoFeedCacheDbContext>();
var results = await feedProvider.GetGeoFeedData();
results.ForEach(x =>
{
geoFeedCacheEntry.Add(new()
{
Prefix = x.Prefix,
GeolocCity = x.GeolocCity,
GeolocCountry = x.GeolocCountry,
GeolocHasLocation = x.GeolocHasLocation,
GeolocPostalCode = x.GeolocPostalCode,
GeolocRegion = x.GeolocRegion
});
});
if (dbContext.GeoFeedCacheEntries.Any())
{
dbContext.GeoFeedCacheEntries.RemoveRange(dbContext.GeoFeedCacheEntries.ToArray());
}
await dbContext.AddRangeAsync(geoFeedCacheEntry, Token);
await dbContext.SaveChangesAsync(Token);
}
catch (Exception ex)
{
logger.LogWarning("On disk cache failed to run. Waiting on 30 minutes before retry...");
}
await Task.Delay(TimeSpan.FromMinutes(30));
}
return false;
}
private async Task DBContextMigrate()
{
using IServiceScope scope = host.Services.CreateScope();
using GeoFeedCacheDbContext? dbContext =
scope.ServiceProvider.GetService<GeoFeedCacheDbContext>();
#pragma warning disable CS8602 // Dereference of a possibly null reference.
if (dbContext.Database.GetPendingMigrations().Any()) {
await dbContext.Database.MigrateAsync();
}
#pragma warning restore CS8602 // Dereference of a possibly null reference.
}
}
}

View File

@ -1,7 +1,9 @@
using AS1024.GeoFeed.GeoFeedBuilder;
using AS1024.GeoFeed.GeoFeedLocalCache;
using AS1024.GeoFeed.Interfaces;
using AS1024.GeoFeed.Core.Interfaces;
using AS1024.GeoFeed.Core.GeoFeedPreloader;
using AS1024.GeoFeed.Core.GeoFeedProviders;
using Microsoft.EntityFrameworkCore;
using AS1024.GeoFeed.Core.GeoFeedSqliteLocalCache;
using AS1024.GeoFeed.Core.CacheService;
namespace AS1024.GeoFeed
{
@ -13,11 +15,13 @@ namespace AS1024.GeoFeed
builder.Services.AddHostedService<PreLoadGeoFeed>();
builder.Services.AddTransient<IGeoFeedProvider, NetBoxGeoFeedProvider>();
builder.Services.AddDbContext<GeoFeedCacheDbContext>(
options =>
{
options.UseSqlite(builder.Configuration.GetConnectionString("LocalFeedCache"));
});
builder.Services.AddScoped<IGeoFeedPersistentCacheProvider, GeoFeedSqliteCache>();
builder.Services.AddHostedService<GeoFeedCacheService>();
builder.Services.AddDbContext<GeoFeedCacheDbContext>(options =>
{
options.UseSqlite(builder.Configuration.GetConnectionString("LocalFeedCache"));
});
builder.Services.AddHttpClient();
builder.Services.AddMemoryCache();
// Add services to the container.

View File

@ -4,25 +4,46 @@
This web application provides a self-published IP geolocation feed, conforming to the standards set out in [RFC 8805](https://www.rfc-editor.org/rfc/rfc8805). It is designed to interface with NetBox, a web-based infrastructure resource modeling (IRM) tool, to retrieve and format geolocation data for IP addresses.
The application is implemented in C# using .NET 8.0, ensuring a robust and modern back-end architecture. It's built to communicate securely over HTTPS with the NetBox API, adhering to best practices for data transmission and security.
The application is implemented in C# using .NET 8.0, ensuring a robust and modern back-end architecture. It's built to communicate securely over HTTPS with the NetBox API, adhering to best practices for data transmission and security.
This project was inspired by the GitHub project [GeoBox](https://github.com/FrumentumNL/GeoBox), which provided foundational ideas for the development of this application.
## Application Variants
### AS1024.GeoFeed - MVC Version (Recommended)
The standard version of this application, named `AS1024.GeoFeed`, is built using the Model-View-Controller (MVC) architecture. This variant is fully supported and recommended for most use cases. It offers a complete set of features and is optimized for robustness and scalability.
### AS1024.GeoFeed.MinimalAPI - MinimalAPI Version (Experimental)
In addition to the standard MVC version, there is an experimental MinimalAPI version of the application named `AS1024.GeoFeed.MinimalAPI`. This variant is designed for environments that require extremely fast startup times, such as serverless containers. However, it comes with limited support and a reduced feature set. We recommend using the `AS1024.GeoFeed.MinimalAPI` version only if your deployment environment necessitates near-instant startup times and you are comfortable with its experimental nature and limitations.
**Note**: While the `AS1024.GeoFeed.MinimalAPI` version offers performance benefits in specific scenarios, we strongly recommend deploying the `AS1024.GeoFeed` MVC version for most applications to ensure full functionality and support.
## Features
- **GeoFeed Generation**: Dynamically generates a geolocation feed in CSV format as specified in RFC 8805.
- **Caching Mechanism**: Implements an efficient caching strategy to reduce redundant API calls and enhance performance.
- **Local Disk Fallback Caching Mechanism**: In the event of a failure to communicate with the NetBox instance specified, the web app will return data that is locally cached inside a SQLite database.
- **Local Disk Fallback Caching Mechanism**: In the event of a failure to communicate with the NetBox instance specified, the web app will return data that is locally cached inside a SQLite database. In the minimal API version of the GeoFeed application, this will be stored as a file, rather than a SQLite database as Minimal API does not yet support Entity Framework Core at this time.
- **Secure Communication**: Ensures secure data retrieval from NetBox over HTTPS.
## Configuration
The application requires the following configuration variables to be set:
The application requires the following configuration variables to be set, depending on the version you are using:
### For AS1024.GeoFeed - MVC Version
1. **APIKey**: This is the API key used for authenticating with the NetBox API. Ensure this key has the necessary permissions to access the required resources.
2. **NetBoxHost**: The hostname of the NetBox instance from which the application retrieves data. For example, `netbox.example.com`.
3. **LocalFeedCache**: This connection string is for a local SQLite Database that caches the geofeed data from Netbox.
3. **LocalFeedCache**: This connection string is for a local SQLite Database (using EF Core) that caches the geofeed data from Netbox. The syntax should follow the standard SQLite EF Core connection string format.
### For AS1024.GeoFeed.MinimalAPI - MinimalAPI Version
1. **APIKey**: This is the API key used for authenticating with the NetBox API. Ensure this key has the necessary permissions to access the required resources.
2. **NetBoxHost**: The hostname of the NetBox instance from which the application retrieves data. For example, `netbox.example.com`.
3. **TempCache** (optional): This configuration is specific to the Minimal API version. It is optional, but if users wish to specify a different location for storing and serving the cached geofeed data, this value can be adjusted to point to the desired save location.
These variables can be set in your application's configuration file or through environment variables, depending on your deployment strategy.
## NetBox Custom Fields
Ensure that your NetBox instance is configured with the following custom fields:
@ -46,6 +67,37 @@ To build and run the application, follow these steps:
5. After a successful build, you can start the application by running `dotnet run`.
6. The application will start, and you can access the endpoints as specified.
## Building a Docker Image
### Building a Docker Image for the Minimal API Version
To containerize the `AS1024.GeoFeed` version using Docker, you can follow these steps to build a Docker image:
1. Open your terminal or command line interface.
2. Navigate to the root directory of this repository.
3. Execute the following Docker build command: ``docker buildx build --platform=linux/amd64,linux/arm64 -f .\AS1024.GeoFeed\Dockerfile .``
### Building a Docker Image for the Minimal API Version
To containerize the `AS1024.GeoFeed.MinimalAPI` version using Docker, you can follow these steps to build a Docker image:
1. Open your terminal or command line interface.
2. Navigate to the root directory of this repository.
3. Execute the following Docker build command: ``docker buildx build --platform=linux/amd64 -f .\AS1024.GeoFeed.MinimalAPI\Dockerfile.alpine-selfcontained .``
**Currently the minimal API version does not support being cross compiled for different CPU architectures. It is best built with the same CPU architecture where the build machine is targeting to.**
Ensure you have Docker installed and configured on your machine before executing this command. This Docker image can then be used to deploy the application in containerized environments such as Kubernetes or Docker Compose setups.
## Docker Deployment
To deploy the Docker container of this GeoFeed application, simply pull the image for the following variants:
- MVC Variant ``docker pull git.startmywifi.com/as1024/geofeed:latest``
- Minimal API Variant ``docker pull git.startmywifi.com/as1024/geofeed:aot-minimal``
To configure the container, please refer to the Configuration section above. You should be able to configure this with either an environment variable or map an ``appSettings.json`` file to the container.
## Endpoints
The application provides the following key endpoints: