Merge branch 'master' of ssh://git.startmywifi.com:2251/AS1024/GeoFeed

AS1024.GeoFeed/AS1024.GeoFeed.csproj

@@ -9,6 +9,15 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.0">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.0" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
   </ItemGroup>
 

AS1024.GeoFeed/Controllers/GeofeedController.cs

@@ -4,6 +4,7 @@ using AS1024.GeoFeed.GeoFeedBuilder;
 using Microsoft.Extensions.Caching.Memory;
 using AS1024.GeoFeed.Models;
 using System.Text;
+using AS1024.GeoFeed.GeoFeedLocalCache;
 
 namespace AS1024.GeoFeed.Controllers
 {
@@ -17,13 +18,16 @@ namespace AS1024.GeoFeed.Controllers
         private readonly IMemoryCache memoryCache;
         private readonly IWebHostEnvironment environment;
         private readonly ILogger<GeofeedController> logger;
+        private readonly GeoFeedCacheDbContext dbContext;
         private const string GeoFeedCacheKey = "GeoFeedData";
 
         public GeofeedController(IGeoFeedProvider builder,
             IMemoryCache memoryCache,
             IWebHostEnvironment environment,
-            ILogger<GeofeedController> logger) {
+            ILogger<GeofeedController> logger,
+            GeoFeedCacheDbContext dbContext) {
             this.logger = logger;
+            this.dbContext = dbContext;
             this.builder = builder;
             this.memoryCache = memoryCache;
             this.environment = environment;
@@ -46,19 +50,36 @@ namespace AS1024.GeoFeed.Controllers
                     }
                 }
 
-                string csvContent = feed.ToGeoFeedCsv(); // Assuming ToGeoFeedCsv() returns a string in CSV format.
+                return ReturnFile(feed);
-                byte[] contentBytes = Encoding.UTF8.GetBytes(csvContent);
+            } catch (HttpRequestException ex)
-                string contentType = "text/csv";
+            {
+                logger.LogWarning($"Temporary failure of retrieving GeoData from upstream. {ex}");
+                var results = 
+                    dbContext.GeoFeedCacheEntries.ToList();
+                List<IPGeoFeed> cachedData = [];
+                results.ForEach(cachedData.Add);
+                return ReturnFile(cachedData);
+            }
 
-                return new FileContentResult(contentBytes, contentType)
+            catch (Exception ex)
-                {
-                    FileDownloadName = "geofeed.csv"
-                };
-            } catch (Exception ex)
             {
                 logger.LogError($"Geofeed generation failed. Exception: {ex}");
                 return StatusCode(500);
             }
+
+        }
+
+        [NonAction]
+        private IActionResult ReturnFile(List<IPGeoFeed>? feed)
+        {
+            string csvContent = feed.ToGeoFeedCsv(); // Assuming ToGeoFeedCsv() returns a string in CSV format.
+            byte[] contentBytes = Encoding.UTF8.GetBytes(csvContent);
+            string contentType = "text/csv";
+
+            return new FileContentResult(contentBytes, contentType)
+            {
+                FileDownloadName = "geofeed.csv"
+            };
         }
     }
 }

AS1024.GeoFeed/GeoFeedBuilder/NetBoxGeoFeedProvider.cs

@@ -31,7 +31,7 @@ namespace AS1024.GeoFeed.GeoFeedBuilder
 
         public async Task<List<IPGeoFeed>> GetGeoFeedData()
         {
-            List<IPGeoFeed> geoFeed = new List<IPGeoFeed>();
+            List<IPGeoFeed> geoFeed = [];
             using HttpClient client = httpClientFactory.CreateClient();
             client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", configuration["APIKey"]);
             client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
@@ -46,45 +46,47 @@ namespace AS1024.GeoFeed.GeoFeedBuilder
                     logger.LogDebug($"Making request to {uri}...");
 
                     using HttpResponseMessage result = await client.GetAsync(uri);
-                    if (result.IsSuccessStatusCode)
+                    if (!result.IsSuccessStatusCode)
                     {
-                        string stringResult = await result.Content.ReadAsStringAsync();
+                        break;
-                        jsonData = JsonSerializer.Deserialize<NetboxData>(stringResult, new JsonSerializerOptions { 
+                    }
-                            PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
+                    string stringResult = await result.Content.ReadAsStringAsync();
-                        });
+                    jsonData = JsonSerializer.Deserialize<NetboxData>(stringResult, new JsonSerializerOptions
+                    {
+                        PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
+                    });
 
-                        if (jsonData?.Results == null || jsonData.Results.Count == 0)
+                    if (jsonData?.Results == null || jsonData.Results.Count == 0)
-                        {
+                    {
-                            break;
+                        break;
-                        }
+                    }
 
-                        foreach (Result data in jsonData.Results)
+                    foreach (Result data in jsonData.Results)
+                    {
+                        try
                         {
-                            try
+                            geoFeed.Add(new IPGeoFeed
                             {
-                                geoFeed.Add(new IPGeoFeed
+                                Prefix = data.Prefix,
-                                {
+                                GeolocCity = data.CustomFields.GeolocCity,
-                                    Prefix = data.Prefix,
+                                GeolocRegion = data.CustomFields.GeolocRegion,
-                                    GeolocCity = data.CustomFields.GeolocCity,
+                                GeolocCountry = data.CustomFields.GeolocCountry,
-                                    GeolocRegion = data.CustomFields.GeolocRegion,
+                                GeolocHasLocation = data.CustomFields.GeolocHasLocation,
-                                    GeolocCountry = data.CustomFields.GeolocCountry,
+                                GeolocPostalCode = data.CustomFields.GeolocPostalCode
-                                    GeolocHasLocation = data.CustomFields.GeolocHasLocation,
+                            });
-                                    GeolocPostalCode = data.CustomFields.GeolocPostalCode
-                                });
-                            }
-                            catch (Exception ex)
-                            {
-                                logger.LogError(ex.ToString());
-                            }
                         }
-
+                        catch (Exception ex)
-                        if (!string.IsNullOrEmpty(jsonData.Next))
                         {
-                            uri = new Uri(jsonData.Next);
+                            logger.LogError(ex.ToString());
-                            continue;
                         }
                     }
 
+                    if (!string.IsNullOrEmpty(jsonData.Next))
+                    {
+                        uri = new Uri(jsonData.Next);
+                        continue;
+                    }
+
                     break;
                 }
             }

AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedCacheDbContext.cs

@@ -0,0 +1,15 @@
+using Microsoft.EntityFrameworkCore;
+
+namespace AS1024.GeoFeed.GeoFeedLocalCache
+{
+    public class GeoFeedCacheDbContext : DbContext
+    {
+        public GeoFeedCacheDbContext(DbContextOptions options)
+            : base(options)
+        {
+        }
+
+        public virtual DbSet<GeoFeedCacheEntry> GeoFeedCacheEntries { get; set; }
+    }
+
+}

AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedCacheEntry.cs

@@ -0,0 +1,12 @@
+using AS1024.GeoFeed.Models;
+using System.ComponentModel.DataAnnotations;
+
+namespace AS1024.GeoFeed.GeoFeedLocalCache
+{
+    public class GeoFeedCacheEntry : IPGeoFeed
+    {
+        [Key]
+        public int Id { get; set; }
+    }
+
+}

AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedCacheService.cs

@@ -0,0 +1,90 @@
+using AS1024.GeoFeed.Interfaces;
+using Microsoft.EntityFrameworkCore;
+using System.Runtime.CompilerServices;
+
+namespace AS1024.GeoFeed.GeoFeedLocalCache
+{
+    public class GeoFeedCacheService : IHostedService
+    {
+        private readonly ILogger<GeoFeedCacheService> logger;
+        private readonly IGeoFeedProvider feedProvider;
+        private readonly IHost host;
+
+        public GeoFeedCacheService(ILogger<GeoFeedCacheService> logger,
+        IGeoFeedProvider feedProvider,
+        IHost host)
+        {
+            this.logger = logger;
+            this.feedProvider = feedProvider;
+            this.host = host;
+        }
+
+        public Task StartAsync(CancellationToken cancellationToken)
+        {
+            _ = StartPerioidicSync(cancellationToken);
+            return Task.CompletedTask;
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            return Task.CompletedTask;
+        }
+
+        public async Task<bool> StartPerioidicSync(CancellationToken Token)
+        {
+            await DBContextMigrate();
+
+            List<GeoFeedCacheEntry> geoFeedCacheEntry = [];
+            while (!Token.IsCancellationRequested)
+            {
+                logger.LogInformation("Running on disk fallback cache process...");
+                try
+                {
+                    using var scope = host.Services.CreateScope();
+                    using var dbContext = scope.ServiceProvider.GetRequiredService<GeoFeedCacheDbContext>();
+                    var results = await feedProvider.GetGeoFeedData();
+
+                    results.ForEach(x =>
+                    {
+                        geoFeedCacheEntry.Add(new()
+                        {
+                            Prefix = x.Prefix,
+                            GeolocCity = x.GeolocCity,
+                            GeolocCountry = x.GeolocCountry,
+                            GeolocHasLocation = x.GeolocHasLocation,
+                            GeolocPostalCode = x.GeolocPostalCode,
+                            GeolocRegion = x.GeolocRegion
+                        });
+                    });
+
+                    if (dbContext.GeoFeedCacheEntries.Any())
+                    {
+                        dbContext.GeoFeedCacheEntries.RemoveRange(dbContext.GeoFeedCacheEntries.ToArray());
+                    }
+                    await dbContext.AddRangeAsync(geoFeedCacheEntry, Token);
+                    await dbContext.SaveChangesAsync(Token);
+                }
+                catch (Exception ex)
+                {
+                    logger.LogWarning("On disk cache failed to run. Waiting on 30 minutes before retry...");
+                }
+                await Task.Delay(TimeSpan.FromMinutes(30));
+            }
+
+            return false;
+        }
+
+        private async Task DBContextMigrate()
+        {
+            using IServiceScope scope = host.Services.CreateScope();
+            using GeoFeedCacheDbContext? dbContext =
+                scope.ServiceProvider.GetService<GeoFeedCacheDbContext>();
+#pragma warning disable CS8602 // Dereference of a possibly null reference.
+            if (dbContext.Database.GetPendingMigrations().Any()) {
+                await dbContext.Database.MigrateAsync();
+            }
+#pragma warning restore CS8602 // Dereference of a possibly null reference.
+
+        }
+    }
+}

AS1024.GeoFeed/GeoFeedLocalCache/GeoFeedDesignTimeMigration.cs

@@ -0,0 +1,16 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Design;
+
+namespace AS1024.GeoFeed.GeoFeedLocalCache
+{
+    public class GeoFeedDesignTimeMigration : IDesignTimeDbContextFactory<GeoFeedCacheDbContext>
+    {
+        public GeoFeedCacheDbContext CreateDbContext(string[] args)
+        {
+            var builder = new DbContextOptionsBuilder<GeoFeedCacheDbContext>();
+            builder.UseSqlite("Data Source=migratedb.db");
+            return new GeoFeedCacheDbContext(builder.Options);
+        }
+    }
+
+}

AS1024.GeoFeed/Migrations/20240108180753_DiskCacheMigration.Designer.cs

@@ -0,0 +1,54 @@
+// <auto-generated />
+using System;
+using AS1024.GeoFeed.GeoFeedLocalCache;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+
+#nullable disable
+
+namespace AS1024.GeoFeed.Migrations
+{
+    [DbContext(typeof(GeoFeedCacheDbContext))]
+    [Migration("20240108180753_DiskCacheMigration")]
+    partial class DiskCacheMigration
+    {
+        /// <inheritdoc />
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
+
+            modelBuilder.Entity("AS1024.GeoFeed.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("GeolocCity")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("GeolocCountry")
+                        .HasColumnType("TEXT");
+
+                    b.Property<bool?>("GeolocHasLocation")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("GeolocPostalCode")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("GeolocRegion")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("Prefix")
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("GeoFeedCacheEntries");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

AS1024.GeoFeed/Migrations/20240108180753_DiskCacheMigration.cs

@@ -0,0 +1,39 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace AS1024.GeoFeed.Migrations
+{
+    /// <inheritdoc />
+    public partial class DiskCacheMigration : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "GeoFeedCacheEntries",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    GeolocCity = table.Column<string>(type: "TEXT", nullable: true),
+                    GeolocCountry = table.Column<string>(type: "TEXT", nullable: true),
+                    GeolocHasLocation = table.Column<bool>(type: "INTEGER", nullable: true),
+                    GeolocRegion = table.Column<string>(type: "TEXT", nullable: true),
+                    GeolocPostalCode = table.Column<string>(type: "TEXT", nullable: true),
+                    Prefix = table.Column<string>(type: "TEXT", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_GeoFeedCacheEntries", x => x.Id);
+                });
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "GeoFeedCacheEntries");
+        }
+    }
+}

AS1024.GeoFeed/Migrations/GeoFeedCacheDbContextModelSnapshot.cs

@@ -0,0 +1,51 @@
+// <auto-generated />
+using System;
+using AS1024.GeoFeed.GeoFeedLocalCache;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+
+#nullable disable
+
+namespace AS1024.GeoFeed.Migrations
+{
+    [DbContext(typeof(GeoFeedCacheDbContext))]
+    partial class GeoFeedCacheDbContextModelSnapshot : ModelSnapshot
+    {
+        protected override void BuildModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder.HasAnnotation("ProductVersion", "8.0.0");
+
+            modelBuilder.Entity("AS1024.GeoFeed.GeoFeedLocalCache.GeoFeedCacheEntry", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("GeolocCity")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("GeolocCountry")
+                        .HasColumnType("TEXT");
+
+                    b.Property<bool?>("GeolocHasLocation")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("GeolocPostalCode")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("GeolocRegion")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("Prefix")
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("GeoFeedCacheEntries");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

AS1024.GeoFeed/Program.cs

@@ -1,5 +1,7 @@
 using AS1024.GeoFeed.GeoFeedBuilder;
+using AS1024.GeoFeed.GeoFeedLocalCache;
 using AS1024.GeoFeed.Interfaces;
+using Microsoft.EntityFrameworkCore;
 
 namespace AS1024.GeoFeed
 {
@@ -11,6 +13,11 @@ namespace AS1024.GeoFeed
 
             builder.Services.AddHostedService<PreLoadGeoFeed>();
             builder.Services.AddTransient<IGeoFeedProvider, NetBoxGeoFeedProvider>();
+            builder.Services.AddHostedService<GeoFeedCacheService>();
+            builder.Services.AddDbContext<GeoFeedCacheDbContext>(options =>
+            {
+                options.UseSqlite(builder.Configuration.GetConnectionString("LocalFeedCache"));
+            });
             builder.Services.AddHttpClient();
             builder.Services.AddMemoryCache();
             // Add services to the container.

AS1024.GeoFeed/appsettings.json

@@ -5,6 +5,9 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
+  "ConnectionStrings": {
+    "LocalFeedCache": "Data Source=localcache.db"
+  },
   "AllowedHosts": "*",
   "APIKey": "",
   "NetBoxHost": ""

LICENSE.txt

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) [year], [fullname]
+Copyright (c) 2024, 12393239 Canada Inc. 
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

README.md

@@ -10,6 +10,7 @@ The application is implemented in C# using .NET 8.0, ensuring a robust and moder
 
 - **GeoFeed Generation**: Dynamically generates a geolocation feed in CSV format as specified in RFC 8805.
 - **Caching Mechanism**: Implements an efficient caching strategy to reduce redundant API calls and enhance performance.
+- **Local Disk Fallback Caching Mechanism**: In the event of a failure to communicate with the NetBox instance specified, the web app will return data that is locally cached inside a SQLite database.
 - **Secure Communication**: Ensures secure data retrieval from NetBox over HTTPS.
 
 ## Configuration
@@ -18,9 +19,22 @@ The application requires the following configuration variables to be set:
 
 1. **APIKey**: This is the API key used for authenticating with the NetBox API. Ensure this key has the necessary permissions to access the required resources.
 2. **NetBoxHost**: The hostname of the NetBox instance from which the application retrieves data. For example, `netbox.example.com`.
+3. **LocalFeedCache**: This connection string is for a local SQLite Database that caches the geofeed data from Netbox.
 
 These variables can be set in your application's configuration file or through environment variables, depending on your deployment strategy.
 
+## NetBox Custom Fields
+
+Ensure that your NetBox instance is configured with the following custom fields:
+
+- `geoloc_city`: (Text) Represents the city and is not required to be filled in.
+- `geoloc_country`: (Selection) Represents the country and is not required to be filled in.
+- `geoloc_has_location`: (Boolean) Indicates if there is geolocation data available and is required.
+- `geoloc_postal_code`: (Text) Represents the postal code and is not required to be filled in.
+- `geoloc_region`: (Selection) Represents the region and is not required to be filled in.
+
+These fields are critical for the application to accurately retrieve and format geolocation data.
+
 ## Getting Started
 
 To build and run the application, follow these steps:
@@ -32,10 +46,6 @@ To build and run the application, follow these steps:
 5. After a successful build, you can start the application by running `dotnet run`.
 6. The application will start, and you can access the endpoints as specified.
 
-## Docker Build
-
-A Dockerfile is provided for your convienence. This has not been tested as of yet.
-
 ## Endpoints
 
 The application provides the following key endpoints: