Gremlinsoft
/
Gremlin
2
0
Fork 0
Code Issues 1 Pull Requests Packages Projects 1 Releases Wiki Activity

refactoring GenericImporter

pull/1/head
Sascha Woitschetzki 2023-07-06 13:55:56 +07:00
parent 5fb9315591
commit 826e274782
7 changed files with 179 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gremlin_BlazorServer/Data/EntityClasses/Contact.cs
Unescape Escape View File

@ -16,7 +16,7 @@ public class Contact : IMetadata {
public string? AcademicTitle { get; set; }
public string? FirstName { get; set; }
public string? LastName { get; set; }
public byte Gender { get; set; }
public byte Gender { get; set; } = 0;
public bool OptInStatus { get; set; }
public string? Department { get; set; }
public string? Function { get; set; }

2
Gremlin_BlazorServer/Pages/Accounts.razor.cs
Unescape Escape View File

@ -45,7 +45,7 @@ public partial class Accounts {
_ = stream.Seek(0, SeekOrigin.Begin);
using StreamReader reader = new(stream);
string fileContent = await reader.ReadToEndAsync();
_ = await GenericImporter.ImportCsvAsync<Account>(fileContent);
await GenericImporter.ImportCsvAsync<Account>(fileContent);
}
}
catch (Exception exception) {

2
Gremlin_BlazorServer/Pages/Contacts.razor.cs
Unescape Escape View File

@ -80,7 +80,7 @@ public partial class Contacts {
_ = stream.Seek(0, SeekOrigin.Begin);
using StreamReader reader = new(stream);
string fileContent = await reader.ReadToEndAsync();
_ = await GenericImporter.ImportCsvAsync<Account>(fileContent);
await GenericImporter.ImportCsvAsync<Contact>(fileContent);
}
}
catch (Exception exception) {

2
Gremlin_BlazorServer/Pages/CustomDescriptions.razor.cs
Unescape Escape View File

@ -34,7 +34,7 @@ public partial class CustomDescriptions {
_ = stream.Seek(0, SeekOrigin.Begin);
using StreamReader reader = new(stream);
string fileContent = await reader.ReadToEndAsync();
_ = await GenericImporter.ImportCsvAsync<CustomDescription>(fileContent);
await GenericImporter.ImportCsvAsync<CustomDescription>(fileContent);
}
}
catch (Exception exception) {

61
Gremlin_BlazorServer/Services/GenericController.cs
Unescape Escape View File

@ -140,36 +140,15 @@ public class GenericController {
}
}
private static void HandleConcurrencyExceptions(DbUpdateException ex) {
Debug.WriteLine(ex.InnerException);
foreach (EntityEntry entry in ex.Entries) {
switch (entry.Entity) {
case Quote or CustomDescription: {
PropertyValues proposedValues = entry.CurrentValues;
PropertyValues? databaseValues = entry.GetDatabaseValues();
foreach (IProperty property in proposedValues.Properties) {
object? proposedValue = proposedValues[property];
object? databaseValue = databaseValues[property];
// TODO: decide which value should be written to database
// proposedValues[property] = <>;
}
// Refresh original values to bypass next concurrency check
entry.OriginalValues.SetValues(proposedValues);
break;
}
default:
throw new NotSupportedException("Don't know how to handle concurrency conflicts for " + entry.Metadata.Name);
}
}
}
public static async Task<int> InsertAsync<T>(IEnumerable<T> entities) where T : class, IMetadata {
try {
gremlinDb.Set<T>().AddRange(entities);
return await gremlinDb.SaveChangesAsync();
}
catch (DbUpdateException ex) {
HandleConcurrencyExceptions(ex);
return -1;
}
catch (Exception exception) {
Console.WriteLine(exception.InnerException);
return 0;
@ -191,6 +170,10 @@ public class GenericController {
gremlinDb.Set<T>().Update(entity);
return await gremlinDb.SaveChangesAsync(false);
}
catch (DbUpdateException ex) {
HandleConcurrencyExceptions(ex);
return -1;
}
catch (Exception exception) {
Console.WriteLine(exception.InnerException);
return 0;
@ -202,6 +185,10 @@ public class GenericController {
await Task.Run(() => gremlinDb.Set<T>().UpdateRange(entities));
return await gremlinDb.SaveChangesAsync();
}
catch (DbUpdateException ex) {
HandleConcurrencyExceptions(ex);
return -1;
}
catch (Exception exception) {
Console.WriteLine(exception.InnerException);
return 0;
@ -218,4 +205,28 @@ public class GenericController {
return 0;
}
}
private static void HandleConcurrencyExceptions(DbUpdateException ex) {
foreach (EntityEntry entry in ex.Entries) {
switch (entry.Entity) {
case Quote or CustomDescription or Account or Contact: {
PropertyValues proposedValues = entry.CurrentValues;
PropertyValues? databaseValues = entry.GetDatabaseValues();
foreach (IProperty property in proposedValues.Properties) {
object? proposedValue = proposedValues[property];
object? databaseValue = databaseValues[property];
// TODO: decide which value should be written to database
// proposedValues[property] = <>;
}
// Refresh original values to bypass next concurrency check
entry.OriginalValues.SetValues(proposedValues);
break;
}
default:
throw new NotSupportedException("Don't know how to handle concurrency conflicts for " + entry.Metadata.Name);
}
}
}
}

225
Gremlin_BlazorServer/Services/GenericImporter.cs
Unescape Escape View File

@ -1,38 +1,17 @@
using Gremlin_BlazorServer.Data.EntityClasses;
using Gremlin_BlazorServer.Pages;
using Blazorise;
using Gremlin_BlazorServer.Data.EntityClasses;
using MySqlX.XDevAPI.Common;
namespace Gremlin_BlazorServer.Services;
public class GenericImporter {
private readonly List<Account> newAccounts = new();
private readonly List<Account> updatedAccounts = new();
public async Task<bool> ImportCsvAsync<T>(string fileContent) where T : class, IMetadata {
Console.WriteLine("GENERIC IMPORTER: Importing accounts from csv...");
newAccounts.Clear();
updatedAccounts.Clear();
public static async Task ImportCsvAsync<T>(string fileContent) where T : class, IMetadata {
Console.WriteLine($"GENERIC IMPORTER: Importing {typeof(T)} from csv...");
List<string[]> splitLines = await Task.Run(() => SplitLines(fileContent));
if (typeof(T) != typeof(Account)) return false;
Console.WriteLine($"Found {splitLines.Count} potential accounts in csv.");
await ParseListToAccounts(splitLines);
int countNewAccounts = 0;
if (newAccounts.Count > 0) {
Console.WriteLine($"There are {newAccounts.Count} new Accounts for the database.");
countNewAccounts = await GenericController.InsertAsync(newAccounts);
}
int countUpdatedAccounts = 0;
if (updatedAccounts.Count > 0) {
Console.WriteLine($"There are {newAccounts.Count} updated Accounts for the database.");
countUpdatedAccounts = await GenericController.UpdateAsync(updatedAccounts);
}
Console.WriteLine($"SUMMARY: Added {countNewAccounts} new Accounts to database and updated {countUpdatedAccounts} Accounts.");
return countNewAccounts > 0 || updatedAccounts.Count > 0;
Console.WriteLine($"Found {splitLines.Count} potential {typeof(T)} in csv.");
await ParseLinesToResultSet<T>(splitLines);
Console.WriteLine($"GENERIC IMPORTER: Ready!");
}
private static List<string[]> SplitLines(string fileContent) {
@ -41,67 +20,141 @@ public class GenericImporter {
return fileList;
}
private async Task ParseListToAccounts(List<string[]> lineList) // ID;Acct Name 1 and 2;Street;City;BP Role;Postal Code;Customer Type;Market Indicator;
private static async Task ParseLinesToResultSet<T>(IEnumerable<string[]> lineList) where T : class, IMetadata
{
int i = 0;
int maxI = lineList.Count;
Result<T>? result = null;
foreach (string[] line in lineList.Select(strings => strings.Select(x => x.Replace("\"", string.Empty)).ToArray())) { // Delete all ""
if (typeof(T) == typeof(Account)) result = await ParseToAccount(line) as Result<T>;
if (typeof(T) == typeof(Contact)) result = await ParseToContact(line) as Result<T>;
if (result is null) continue;
if (result.NewItem is not null) await GenericController.InsertAsync(result.NewItem);
if (result.UpdatedItem is not null) await GenericController.UpdateAsync(result.UpdatedItem);
}
}
private static async Task<Result<Account>?> ParseToAccount(IReadOnlyList<string> line) { // "ID" "Name" "Street" "City" "BP Role" "Postal Code" "Customer Type" "Market Indicator" "Phone" "E-Mail" "Market code"
if (line[0].Contains("ID")) return null; //HACK: skip first row if header
if (!uint.TryParse(line[0], out uint sapAccountNumber)) return null; //HACK: skip wrong SapAccountNumbers
if (!uint.TryParse(line[5], out uint zip)) return null; //HACK: skip wrong ZIPs
Result<Account> result = new();
foreach (string[] strings in lineList) {
string[] line = strings.Select(x => x.Replace("\"", string.Empty)).ToArray(); // Remove all ""
// Console.WriteLine($"Current line: {line[0]}: {line[1]}...");
i++;
Accounts.ImportProgress = (int)((float)i / maxI * 100);
if (line[0].Contains("ID")) continue; //HACK: skip first row if header
if (!uint.TryParse(line[0], out uint sapAccountNumber)) continue; //HACK: skip wrong SapAccountNumbers
if (!uint.TryParse(line[5], out uint zip)) continue; //HACK: skip wrong ZIPs
string accountTypeCode = line[6];
if (accountTypeCode is "" or null) accountTypeCode = "FPC"; // standard AccountType
string subMarketCode = line[7];
if (subMarketCode is "" or null) subMarketCode = "CCH"; // standard AccountType
Account readAccount = new() {
SapAccountNumber = sapAccountNumber,
AccountName = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToTitleCase(line[1].ToLower()), //line[1]
Street = line[2],
City = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToTitleCase(line[3].ToLower()), //line[3].ToUpper().First() + line[3][1..].ToLower(),
Zip = zip,
PhoneNumber = line[8],
EMail = line[9],
// ParentAccountId = 0,
DataModificationByUser = "Gremlin Generic Importer",
AccountTypeCode = accountTypeCode,
SubMarketCode = subMarketCode
};
if (await GenericController.IsExistingAsync<Account>(a => a.SapAccountNumber.Equals(sapAccountNumber))) {
// Console.WriteLine($"Account {readAccount.AccountName} existiert bereits. Prüfe auf Updates...");
Account? existingAccount = GenericController.Get<Account>(a => a.SapAccountNumber == readAccount.SapAccountNumber);
if (existingAccount is null) continue;
if (IsEqualAccount(readAccount, existingAccount)) continue;
existingAccount.DataModificationDate = DateTime.Now;
existingAccount.DataVersionNumber++;
existingAccount.DataModificationByUser = "Updated by Gremlin Generic Importer";
existingAccount.AccountName = readAccount.AccountName;
existingAccount.City = readAccount.City;
existingAccount.EMail = readAccount.EMail;
existingAccount.PhoneNumber = readAccount.PhoneNumber;
existingAccount.Street = readAccount.Street;
existingAccount.Zip = readAccount.Zip;
Console.WriteLine($"Update in Account {existingAccount.SapAccountNumber}:{existingAccount.AccountName}");
updatedAccounts.Add(existingAccount);
}
else {
Console.WriteLine($"Account {readAccount.SapAccountNumber}:{readAccount.AccountName} ist neu!");
newAccounts.Add(readAccount);
}
string accountTypeCode = line[6];
if (accountTypeCode is "" or null) accountTypeCode = "FPC"; // standard AccountType
string subMarketCode = line[7];
if (subMarketCode is "" or null) subMarketCode = "CCH"; // standard AccountType
Account readAccount = new() {
SapAccountNumber = sapAccountNumber,
AccountName = line[1], //System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToTitleCase(line[1].ToLower())
Street = line[2],
City = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToTitleCase(line[3].ToLower()), //line[3].ToUpper().First() + line[3][1..].ToLower(),
Zip = zip,
PhoneNumber = line[8],
EMail = line[9],
DataModificationByUser = "Gremlin Generic Importer",
AccountTypeCode = accountTypeCode,
SubMarketCode = subMarketCode
};
if (await GenericController.IsExistingAsync<Account>(a => a.SapAccountNumber == sapAccountNumber)) {
// Console.WriteLine($"Account {readAccount.AccountName} existiert bereits. Prüfe auf Updates...");
Account? existingAccount = GenericController.Get<Account>(a => a.SapAccountNumber == sapAccountNumber);
if (existingAccount is null) return null;
if (IsEqualAccount(readAccount, existingAccount)) return null;
existingAccount.DataModificationDate = DateTime.Now;
existingAccount.DataVersionNumber++;
existingAccount.DataModificationByUser = "Updated by Gremlin Generic Importer";
existingAccount.AccountName = readAccount.AccountName;
existingAccount.City = readAccount.City;
existingAccount.EMail = readAccount.EMail;
existingAccount.PhoneNumber = readAccount.PhoneNumber;
existingAccount.Street = readAccount.Street;
existingAccount.Zip = readAccount.Zip;
Console.WriteLine($"Update in Account {existingAccount.SapAccountNumber}:{existingAccount.AccountName}");
result.UpdatedItem = existingAccount;
}
else {
Console.WriteLine($"Account {readAccount.SapAccountNumber}:{readAccount.AccountName} ist neu!");
result.NewItem = readAccount;
}
return result;
}
private static async Task<Result<Contact>?> ParseToContact(IReadOnlyList<string> line) { //"Contact ID" "Account ID" "Last Name" "First Name" "Acct Name 1 and 2" "Street - Work Address" "Postal Code - Work Address" "City - Work Address" "E-Mail" "Phone" "E-Mail Opt"
if (line[0].Contains("ID")) return null; //HACK: skip first row if header
if (!uint.TryParse(line[0], out uint sapContactNumber)) return null; //HACK: skip wrong SapContactNumbers
if (!uint.TryParse(line[1], out uint sapAccountNumber)) return null; //HACK: skip wrong SapAccountNumbers
Account? account = GenericController.Get<Account>(a => a.SapAccountNumber.Equals(sapAccountNumber));
if (account is null) {
Console.WriteLine($"Account with SapAccountNumber {sapAccountNumber} is not existing!!");
return null; //HACK: skip empty Accounts
}
Result<Contact> result = new();
private static bool IsEqualAccount(Account account, Account existingAccount) {
return account.AccountName == existingAccount.AccountName && account.City == existingAccount.City && account.EMail == existingAccount.EMail && account.PhoneNumber == existingAccount.PhoneNumber && account.Street == existingAccount.Street && account.Zip == existingAccount.Zip;
Contact readContact = new() {
SapContactNumber = sapContactNumber,
AccountId = account.AccountId,
LastName = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToTitleCase(line[2].ToLower()),
FirstName = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToTitleCase(line[3].ToLower()),
EMail = line[8].ToLower(),
EmailBounced = line[8].Contains("bounced"),
PhoneNumber = line[9],
OptInStatus = line[10] == "Opt In",
DataModificationByUser = "Gremlin Generic Importer",
Gender = 0
};
if (await GenericController.IsExistingAsync<Contact>(a => a.SapContactNumber == sapContactNumber)) {
Contact? existingContact = GenericController.Get<Contact>(a => a.SapContactNumber == sapContactNumber);
if (existingContact is null) return null;
if (IsEqualContact(readContact, existingContact)) {
Console.WriteLine($"---> The contact {readContact.SapContactNumber}:{readContact.FirstName} {readContact.LastName} ist aktuell.");
return null;
}
existingContact.DataModificationDate = DateTime.Now;
existingContact.DataVersionNumber++;
existingContact.DataModificationByUser = "Updated by Gremlin Generic Importer";
existingContact.AccountId = readContact.AccountId;
existingContact.LastName = readContact.LastName;
existingContact.EMail = readContact.EMail;
existingContact.PhoneNumber = readContact.PhoneNumber;
existingContact.EmailBounced = readContact.EmailBounced;
existingContact.OptInStatus = readContact.OptInStatus;
Console.WriteLine($"Update in Contact {existingContact.SapContactNumber}:{existingContact.FirstName} {existingContact.LastName}");
result.UpdatedItem = existingContact;
}
else {
Console.WriteLine($"Contact {readContact.SapContactNumber}:{readContact.FirstName} {readContact.LastName} ist neu!");
result.NewItem = readContact;
}
return result;
}
private static bool IsEqualAccount(Account account, Account existingAccount)
=> account.AccountName == existingAccount.AccountName
&& account.City == existingAccount.City
&& account.EMail == existingAccount.EMail
&& account.PhoneNumber == existingAccount.PhoneNumber
&& account.Street == existingAccount.Street
&& account.Zip == existingAccount.Zip;
private static bool IsEqualContact(Contact contact, Contact existingContact)
=> contact.LastName == existingContact.LastName
&& contact.FirstName == existingContact.FirstName
&& contact.EMail == existingContact.EMail
&& contact.PhoneNumber == existingContact.PhoneNumber
&& contact.EmailBounced == existingContact.EmailBounced
&& contact.OptInStatus == existingContact.OptInStatus;
}
internal class Result<T> {
public T? NewItem { get; set; }
public T? UpdatedItem { get; set; }
}

BIN
Gremlin_BlazorServer/wwwroot/quotes/RWTH_Aachen_Institut_für_Organische_Chemie/2023-7-Teskey-HPLC+GCMSD+BasicServer/DE-83PE89-723-251.pdf
View File

Binary file not shown.