Merge branch 'master' of https://github.com/ArgumentumGames/Argumentum

ArgumentumGames · Apr 17, 2024 · 8a61aa7 · 8a61aa7
2 parents 8c55221 + ebf2ccb
commit 8a61aa7
Show file tree

Hide file tree

Showing 16 changed files with 1,233 additions and 1,057 deletions.
diff --git a/Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv b/Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv
diff --git a/Cards/Fallacies/Mindmaps/included.html b/Cards/Fallacies/Mindmaps/included.html
@@ -13,10 +13,13 @@
             display: none;
         }
 
-        /* svg {
-            shape-rendering: optimizeSpeed;
-            text-rendering: optimizeSpeed;
-        }*/
+        svg {
+            /* shape-rendering: optimizeSpeed;
+            text-rendering: optimizeSpeed;*/
+            height: 93vh;
+            width: 96vw;
+            max-width: 2000px;
+        }
 
         #mindmap {
             position: relative;
@@ -625,15 +628,15 @@
                     panEnabled: true,
                     mouseWheelZoomEnabled: true,
                     dblClickZoomEnabled: true,
-                    fit: 0,
+                    fit: 1,
                     controlIconsEnabled: true,
-                    minZoom: 0.15,
-                    maxZoom: 6,
-                    center: 0,
+                    minZoom: 1,
+                    maxZoom: 20,
+                    center: 1,
                     //contain: 1,
                     customEventsHandler: eventsHandler
                 });
-
+                panZoomInstance.zoom(5);
                 //panZoomInstance.resize();
                 //panZoomInstance.fit();
                 //panZoomInstance.center();

diff --git a/Generation/Converters/Argumentum.AssetConverter/Argumentum.AssetConverter.csproj b/Generation/Converters/Argumentum.AssetConverter/Argumentum.AssetConverter.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <AssemblyVersion>1.3</AssemblyVersion>
     <FileVersion>1.3</FileVersion>
     <Copyright>Argumentum</Copyright>
@@ -11,23 +11,23 @@
 
   <ItemGroup>
     <PackageReference Include="AutoMapper" Version="13.0.1" />
-    <PackageReference Include="Betalgo.OpenAI" Version="7.4.6" />
-    <PackageReference Include="Betalgo.OpenAI.Utilities" Version="7.0.4" />
-    <PackageReference Include="CsvHelper" Version="31.0.0" />
-    <PackageReference Include="ExtendedXmlSerializer" Version="3.7.11" />
+    <PackageReference Include="Betalgo.OpenAI" Version="8.0.1" />
+    <PackageReference Include="Betalgo.OpenAI.Utilities" Version="8.0.1" />
+    <PackageReference Include="CsvHelper" Version="31.0.3" />
+    <PackageReference Include="ExtendedXmlSerializer" Version="3.7.18" />
     <PackageReference Include="Humanizer" Version="2.14.1" />
     <PackageReference Include="Magick.NET-Q16-AnyCPU" Version="13.6.0" />
-    <PackageReference Include="Microsoft.Playwright" Version="1.41.2" />
+    <PackageReference Include="Microsoft.Playwright" Version="1.43.0" />
     <PackageReference Include="Microsoft.XmlSerializer.Generator" Version="8.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="OWLSharp" Version="3.10.2" />
-    <PackageReference Include="QuestPDF" Version="2023.12.5" />
-    <PackageReference Include="SharpToken" Version="1.2.17" />
+    <PackageReference Include="OWLSharp" Version="3.11.0" />
+    <PackageReference Include="QuestPDF" Version="2024.3.0" />
+    <PackageReference Include="SharpToken" Version="2.0.2" />
     <PackageReference Include="Spectre.Console" Version="0.48.1-preview.0.5" />
     <PackageReference Include="Spectre.Console.Json" Version="0.48.1-preview.0.5" />
     <PackageReference Include="System.ComponentModel.TypeConverter" Version="4.3.0" />
     <PackageReference Include="System.Drawing.Primitives" Version="4.3.0" />
-    <PackageReference Include="System.Linq.Dynamic.Core" Version="1.3.8" />
+    <PackageReference Include="System.Linq.Dynamic.Core" Version="1.3.10" />
     <PackageReference Include="Utf8Json" Version="1.3.7" />
   </ItemGroup>
 
@@ -56,9 +56,15 @@
     <None Update="DatasetUpdater\Resources\PromptInstructionsAssistantDescription.txt">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="DatasetUpdater\Resources\PromptTranslateCleanupInstructionsAssistant.txt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Update="DatasetUpdater\Resources\PromptTranslatePtInstructionsAssistant.txt">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="DatasetUpdater\Resources\PromptTranslateCleanupInstructionsUser.txt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Update="DatasetUpdater\Resources\PromptTranslateRuInstructionsAssistant.txt">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>

diff --git a/Generation/Converters/Argumentum.AssetConverter/AssetConverterConfig.cs b/Generation/Converters/Argumentum.AssetConverter/AssetConverterConfig.cs
@@ -26,9 +26,9 @@ public class AssetConverterConfig
 
 
 		//Debug Switch to configure default values
-	    public bool SkipConfigFile { get; set; } = false;
+	    public bool SkipConfigFile { get; set; } = true;
 
-	    public ConverterMode Mode { get; set; } = ConverterMode.Mindmapper | ConverterMode.OwlGenerator; // ConverterMode.Mindmapper | ConverterMode.OwlGenerator; // | ConverterMode.WebBasedImageGeneration; // ConverterMode.DatasetUpdater;
+	    public ConverterMode Mode { get; set; } = ConverterMode.Mindmapper | ConverterMode.OwlGenerator; // | ConverterMode.WebBasedImageGeneration; // ConverterMode.DatasetUpdater;
 
 		public bool ForceDebugParams { get; set; }
 

diff --git a/Generation/Converters/Argumentum.AssetConverter/BatchImageConverterConfig.cs b/Generation/Converters/Argumentum.AssetConverter/BatchImageConverterConfig.cs
@@ -33,7 +33,7 @@ public async Task Apply()
                     BatchImagePngToCnykJpegsInternal(objSourceDir, objTargetDir);
                     break;
                 case BatchImageOperation.ModulateHue:
-                    BatchImageModulate(objSourceDir, objTargetDir);
+					await BatchImageModulate(objSourceDir, objTargetDir).ConfigureAwait(false);
                     break;
                 default:
                     throw new ArgumentOutOfRangeException();
@@ -43,7 +43,7 @@ public async Task Apply()
 
         }
 
-        private void BatchImageModulate(DirectoryInfo sourceDir, DirectoryInfo targetDir)
+        private async Task BatchImageModulate(DirectoryInfo sourceDir, DirectoryInfo targetDir)
         {
             foreach (var sourceFile in sourceDir.GetFiles())
             {
@@ -55,7 +55,7 @@ private void BatchImageModulate(DirectoryInfo sourceDir, DirectoryInfo targetDir
                         ImageHelper.Modulate(image, Modulation);
 
                         var targetFile = new FileInfo(Path.Combine(targetDir.ToString(), sourceFile.Name));
-                        image.Write(targetFile);
+                        await image.WriteAsync(targetFile).ConfigureAwait(false);
 
                         Logger.LogSuccess($"Image Converted: {targetFile.Directory?.Name}\\{targetFile.Name}");
 

diff --git a/Generation/Converters/Argumentum.AssetConverter/DatasetUpdater/DatasetUpdaterConfig.cs b/Generation/Converters/Argumentum.AssetConverter/DatasetUpdater/DatasetUpdaterConfig.cs
@@ -22,6 +22,8 @@ public class DatasetUpdaterConfig
 
 	public bool Enabled { get; set; } = true;
 
+	public string Name { get; set; } = "Dataset Update";
+
 	public string SystemPromptPath { get; set; }
 
 
@@ -136,7 +138,7 @@ bool RecordHasEmptyTargetFields(Dictionary<string, object> dictionary)
 			DataTable globalDataTable = DataSetInfo.LoadCsvIntoDataTable(content, ",", PrimaryField);
 			var resultDataTablesPerField = new Dictionary<string, DataTable> { };
 			resultDataTablesPerField[""] = globalDataTable;
-
+			List<Dictionary<string, object>> records = null;
 			for (int globalPassIndex = 0; globalPassIndex < this.NbGlobalPasses; globalPassIndex++)
 			{
 				//globalDataTable = DataSetInfo.LoadCsvIntoDataTable(content, ",", PrimaryField);
@@ -149,7 +151,11 @@ bool RecordHasEmptyTargetFields(Dictionary<string, object> dictionary)
 
 					// load dataset in chunks, 
 
-					var records = sourceDataset.GetDictionaryFromCsv(content, FieldsToInclude, config.UseDebugParams);
+					if (records == null)
+					{
+						records = sourceDataset.GetDictionaryFromCsv(content, FieldsToInclude, config.UseDebugParams);
+					}
+
 
 					//if (SelectEmptyTargets)
 					//{
@@ -276,7 +282,6 @@ await Parallel.ForEachAsync(recordGroups, parallelOptions, async (recordGroup, c
 						//		false);
 
 
-
 					}
 
 
@@ -398,7 +403,7 @@ await Parallel.ForEachAsync(recordGroups, parallelOptions, async (recordGroup, c
 
 
 
-				content = DataSetInfo.WriteDataTableToCsv(globalDataTable, ",");
+				//content = DataSetInfo.WriteDataTableToCsv(globalDataTable, ",");
 
 			}
 
@@ -489,15 +494,13 @@ private async Task DoChatGPTCall(CancellationToken ct, List<Dictionary<string, o
 				}
 
 
-
-
 				ct.ThrowIfCancellationRequested();
 				Logger.Log($"Calling ChatGPT API with chunk: \n{Markup.Escape(chunk)}\n");
 
 				try
 				{
 					tokenManager.WaitForTokenAvailability();
-					result = await dataPrompt.Send(token).ConfigureAwait(false);
+					result = await dataPrompt.Send(token, Logger.LogInformation).ConfigureAwait(false);
 					//result = chunk;
 					Logger.Log(
 						$"ChatGPT answered chunk: \n{Markup.Escape(chunk)}\n with chunk \n{Markup.Escape(result)}\n");
@@ -508,7 +511,6 @@ private async Task DoChatGPTCall(CancellationToken ct, List<Dictionary<string, o
 					Logger.LogException(e);
 				}
 
-				List<Dictionary<string, object>> newRecords;
 				if (!UseFunctionCalling)
 				{
 					recordGroup = DataSetInfo.GetDictionaryRecordsFromJson(result);
@@ -522,7 +524,6 @@ private async Task DoChatGPTCall(CancellationToken ct, List<Dictionary<string, o
 
 			}
 
-
 			answers.Add(result);
 		}
 		catch (OperationCanceledException)

diff --git a/Generation/Converters/Argumentum.AssetConverter/DatasetUpdater/DatasetUpdaterRootConfig.cs b/Generation/Converters/Argumentum.AssetConverter/DatasetUpdater/DatasetUpdaterRootConfig.cs
@@ -27,6 +27,7 @@ public async Task Apply(AssetConverterConfig config)
         new DatasetUpdaterConfig()
         {
             Enabled = false,
+			Name = "Update Virtues Taxonomy by chunks 1-shot",
             SourceDataset = KnownDataSets.VirtuesTaxonomy,
             FieldsToInclude = new List<string>()
             {
@@ -70,7 +71,8 @@ public async Task Apply(AssetConverterConfig config)
         new DatasetUpdaterConfig()
         {
             Enabled = false,
-            SourceDataset = KnownDataSets.FallaciesTaxonomy,
+            Name = "Update Fallacies French Description by branch 0-shot",
+			SourceDataset = KnownDataSets.FallaciesTaxonomy,
             FieldsToInclude = new List<string>()
             {
                 "path",
@@ -133,7 +135,8 @@ public async Task Apply(AssetConverterConfig config)
         new DatasetUpdaterConfig()
         {
             Enabled = false,
-            SourceDataset = KnownDataSets.FallaciesTaxonomy,
+            Name = "Update Fallacies French example by branch 0-shot",
+			SourceDataset = KnownDataSets.FallaciesTaxonomy,
             FieldsToInclude = new List<string>()
             {
                 "path",
@@ -197,6 +200,7 @@ public async Task Apply(AssetConverterConfig config)
 		new DatasetUpdaterConfig()
 		{
 			Enabled = false,
+			Name = "Translate Fallacies to English by branch empty-only 0-shot",
 			SourceDataset = KnownDataSets.FallaciesTaxonomy,
 			FieldsToInclude = new List<string>()
 			{
@@ -259,6 +263,7 @@ public async Task Apply(AssetConverterConfig config)
 		new DatasetUpdaterConfig()
 		{
 			Enabled = false,
+			Name = "Translate Fallacies to Russian by chunk empty-only 0-shot",
 			SourceDataset = KnownDataSets.FallaciesTaxonomy,
 			FieldsToInclude = new List<string>()
 			{
@@ -320,7 +325,8 @@ public async Task Apply(AssetConverterConfig config)
 		},
 		new DatasetUpdaterConfig()
 		{
-			Enabled = true,
+			Enabled = false,
+			Name = "Translate Fallacies to Portuguese by chunk empty-only 0-shot",
 			SourceDataset = KnownDataSets.FallaciesTaxonomy,
 			FieldsToInclude = new List<string>()
 			{
@@ -371,7 +377,7 @@ public async Task Apply(AssetConverterConfig config)
 			ChunkSize = 8,
 			SelectEmptyTargets = true,
 			RandomizeChunks = false,
-			MaxDegreeOfParallelismWebService = 7,
+			MaxDegreeOfParallelismWebService = 6,
 			CompareMode = false,
 			AutoCompare = true,
 			AutoCompareField = "text_fr",
@@ -380,6 +386,78 @@ public async Task Apply(AssetConverterConfig config)
 			WriteOneTargetFileByField = false,
 			MaxChildren = 8,
 			NbGlobalPasses = 2
+		},
+		new DatasetUpdaterConfig()
+		{
+			Enabled = true,
+			Name = "Cleanup Fallacies translations by chunk empty-only 0-shot",
+			SourceDataset = KnownDataSets.FallaciesTaxonomy,
+			FieldsToInclude = new List<string>()
+			{
+				"path",
+				"Famille",
+				"Sous-Famille",
+				"Soussousfamille",
+				"text_fr",
+				"desc_fr",
+				"example_fr",
+				//"carte",
+				"link_fr",
+				"text_en",
+				"desc_en",
+				"example_en",
+				"link_en",
+				"text_ru",
+				"desc_ru",
+				"example_ru",
+				"text_pt",
+				"desc_pt",
+				"example_pt",
+			},
+			FieldsToUpdate = new List<string>()
+			{
+				"text_en",
+				"desc_en",
+				"example_en",
+				"text_ru",
+				"desc_ru",
+				"example_ru",
+				"text_pt",
+				"desc_pt",
+				"example_pt"
+			},
+			PrimaryField = "path",
+			TargetPath = @".\Target\Datasets\Argumentum Fallacies - Taxonomy.csv",
+			SystemPromptPath = PromptsRootPath + "PromptGeneralSystem.txt",
+			DialogPrompts = new List<PromptExample>()
+			{
+				new PromptExample()
+				{
+					UserPromptPath = PromptsRootPath + "PromptTranslateCleanupInstructionsUser.txt",
+					AssistantAnswerPath = PromptsRootPath + "PromptTranslateCleanupInstructionsAssistant.txt"
+				}
+			},
+			Model = Models.Gpt_4_turbo,
+			MaxTokensPerMinute = 70000,
+			DivisionMode = DivisionMode.SequentialChunks,
+			PKHierarchyLevel = 3,
+			UseFunctionCalling = true,
+			//FunctionName = nameof(RecordsUpdater.UpdateRecord),
+			NbMessageCalls = 2,
+			SkipChunkNb = 0,
+			TakeChunkNb = -1,
+			ChunkSize = 8,
+			SelectEmptyTargets = false,
+			RandomizeChunks = false,
+			MaxDegreeOfParallelismWebService = 6,
+			CompareMode = false,
+			AutoCompare = true,
+			AutoCompareField = "text_fr",
+			CompareField = "example_fr",
+			MaxGroupItemNb = 12,
+			WriteOneTargetFileByField = true,
+			MaxChildren = 8,
+			NbGlobalPasses = 2
 		}
 
 	};

diff --git a/Generation/Converters/Argumentum.AssetConverter/DatasetUpdater/Prompt.cs b/Generation/Converters/Argumentum.AssetConverter/DatasetUpdater/Prompt.cs
@@ -52,7 +52,7 @@ public OpenAIService OpenAiService
 
 	public string FunctionName { get; set; }
 
-	public async Task<string> Send(CancellationToken cancellationToken)
+	public async Task<string> Send(CancellationToken cancellationToken, Action<string> log)
 	{
 		if (Tokenizer != null)
 		{
@@ -123,6 +123,7 @@ public async Task<string> Send(CancellationToken cancellationToken)
 				{
 					var functionCall = chatMessageToolCall.FunctionCall;
 					var result = CallFunction(functionCall);
+					log($"Function call {functionCall.Name} with arguments {functionCall.Arguments}");
 				}
 			}
 

diff --git a/...m.AssetConverter/DatasetUpdater/Resources/PromptTranslateCleanupInstructionsAssistant.txt b/...m.AssetConverter/DatasetUpdater/Resources/PromptTranslateCleanupInstructionsAssistant.txt
@@ -0,0 +1,2 @@
+J'ai bien pris note des instructions détaillées et je me concentrerai sur les traductions en anglais, en russe et en portugais, en identifiant les entités pour lesquelles une correction ou un ajout est nécessaire, et en proposant les corrections les plus adéquates, pas plus d'une cellule par entité, en prenant soin de prioriser les modifications, exemple, puis description, puis titre, pour conserver la meilleure intégrité. Je ne chercherai pas à répondre textuellement au prochain message que vous allez me soumettre, mais j'utiliserai plutôt la fonctionnalité de function-calling de l'API OpenAI pour proposer chacune des mes modifications grâce aux fonctions que tu m'as mises à disposition. 
+Prêt à recevoir les données pour la mise à jour.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		J'ai bien pris note des instructions détaillées et je me concentrerai sur les traductions en anglais, en russe et en portugais, en identifiant les entités pour lesquelles une correction ou un ajout est nécessaire, et en proposant les corrections les plus adéquates, pas plus d'une cellule par entité, en prenant soin de prioriser les modifications, exemple, puis description, puis titre, pour conserver la meilleure intégrité. Je ne chercherai pas à répondre textuellement au prochain message que vous allez me soumettre, mais j'utiliserai plutôt la fonctionnalité de function-calling de l'API OpenAI pour proposer chacune des mes modifications grâce aux fonctions que tu m'as mises à disposition.
		Prêt à recevoir les données pour la mise à jour.