Add UseOnDuplicateKeyUpdateForSplitTables

WowPacketParser/App.config

@@ -228,6 +228,15 @@
         -->
         <add key="ForceInsertQueries" value="false"/>
 
+        <!--
+               Option:      UseOnDuplicateKeyUpdateForSplitTables
+               Description: For tables generated in multiple passes (for example creature_template and creature_template_difficulty),
+                            use INSERT ... ON DUPLICATE KEY UPDATE instead of DELETE + INSERT so one pass does not remove
+                            rows/columns generated by another pass.
+               Default:     "false"
+        -->
+        <add key="UseOnDuplicateKeyUpdateForSplitTables" value="false"/>
+
         <!--
                Option:      MaximumConditionsPerStatement
                Description: Defined after how many WHERE conditions a new statement should be created for UPDATE and DELETE FROM

WowPacketParser/Misc/Settings.cs

@@ -27,6 +27,7 @@ public static class Settings
         public static readonly bool SkipRowsWithFallbackValues = Conf.GetBoolean("SkipRowsWithFallbackValues", true);
         public static readonly bool IgnoreZeroValues = Conf.GetBoolean("IgnoreZeroValues", false);
         public static readonly bool ForceInsertQueries = Conf.GetBoolean("ForceInsertQueries", false);
+        public static readonly bool UseOnDuplicateKeyUpdateForSplitTables = Conf.GetBoolean("UseOnDuplicateKeyUpdateForSplitTables", false);
         public static readonly int MaximumConditionsPerStatement = Conf.GetInt("MaximumConditionsPerStatement", 5000);
         public static readonly bool RecalcDiscount = Conf.GetBoolean("RecalcDiscount", false);
         public static readonly bool ForcePhaseZero = Conf.GetBoolean("ForcePhaseZero", false);

WowPacketParser/SQL/Builders/Movement.cs

@@ -177,6 +177,7 @@ protected override string GenerateQuery()
                     }
                 }
 
+                output.ReplaceLastCommaWithSemicolon();
                 output.AppendLine();
 
                 ++pathIdCounter;

WowPacketParser/SQL/Builders/UnitMisc.cs

@@ -155,7 +155,11 @@ public static string CreatureTemplateScalingDataWDB()
 
             var templatesDb = SQLDatabase.Get(Storage.CreatureTemplateDifficultiesWDB);
 
-            return SQLUtil.Compare(Settings.SQLOrderByKey ? Storage.CreatureTemplateDifficultiesWDB.OrderBy(x => x.Item1.Entry).ToArray() : Storage.CreatureTemplateDifficultiesWDB.ToArray(), templatesDb, StoreNameType.Unit);
+            return SQLUtil.Compare(
+                Settings.SQLOrderByKey ? Storage.CreatureTemplateDifficultiesWDB.OrderBy(x => x.Item1.Entry).ToArray() : Storage.CreatureTemplateDifficultiesWDB.ToArray(),
+                templatesDb,
+                StoreNameType.Unit,
+                Settings.UseOnDuplicateKeyUpdateForSplitTables);
         }
 
         public static void UpdateCreatureStaticFlags(ref Unit npc, ref CreatureTemplateDifficulty creatureDifficulty)
@@ -265,7 +269,8 @@ public static string CreatureTemplateScalingData(Dictionary<WowGuid, Unit> units
                         sb.Remove(sb.Length - 3, 3);
 
                     return $"{StoreGetters.GetName(StoreNameType.Unit, (int)x.Entry)} - {sb}";
-                });
+                },
+                Settings.UseOnDuplicateKeyUpdateForSplitTables);
         }
 
         [BuilderMethod(Units = true)]
@@ -955,7 +960,7 @@ public static string CreatureTemplateNonWDB(Dictionary<WowGuid, Unit> units)
             }
 
             var templatesDb = SQLDatabase.Get(Storage.CreatureTemplatesNonWDB);
-            return SQLUtil.Compare(Storage.CreatureTemplatesNonWDB, templatesDb, StoreNameType.Unit);
+            return SQLUtil.Compare(Storage.CreatureTemplatesNonWDB, templatesDb, StoreNameType.Unit, Settings.UseOnDuplicateKeyUpdateForSplitTables);
         }
 
         static UnitMisc()

WowPacketParser/SQL/Builders/WDBTemplates.cs

@@ -103,13 +103,13 @@ public static string CreatureTemplate(Dictionary<WowGuid, Unit> units)
             if (!Storage.CreatureTemplates.IsEmpty() && Settings.TargetedDatabase != TargetedDatabase.Classic)
             {
                 var templatesDb = SQLDatabase.Get(Storage.CreatureTemplates.Values);
-                return SQLUtil.Compare(Storage.CreatureTemplates.Values, templatesDb, StoreNameType.Unit);
+                return SQLUtil.Compare(Storage.CreatureTemplates.Values, templatesDb, StoreNameType.Unit, Settings.UseOnDuplicateKeyUpdateForSplitTables);
             }
 
             if (!Storage.CreatureTemplatesClassic.IsEmpty() && Settings.TargetedDatabase == TargetedDatabase.Classic)
             {
                 var templatesDb = SQLDatabase.Get(Storage.CreatureTemplatesClassic);
-                return SQLUtil.Compare(Storage.CreatureTemplatesClassic, templatesDb, StoreNameType.Unit);
+                return SQLUtil.Compare(Storage.CreatureTemplatesClassic, templatesDb, StoreNameType.Unit, Settings.UseOnDuplicateKeyUpdateForSplitTables);
             }
 
             return string.Empty;

WowPacketParser/SQL/QueryBuilder.cs

@@ -408,7 +408,9 @@ public virtual string Build()
     {
         private readonly RowList<T> _rows;
         private readonly bool _withDelete;
+        private readonly bool _onDuplicateKeyUpdate;
         private readonly string _insertHeader;
+        private readonly string _onDuplicateKeyUpdateClause;
 
         // Add a new insert header every 250 rows
         private const int MaxRowsPerInsert = 250;
@@ -420,11 +422,13 @@ public virtual string Build()
         /// <param name="rows">A list of <see cref="SQLInsertRow{T}"/> rows</param>
         /// <param name="withDelete">If set to false the full query will not include a delete query</param>
         /// <param name="ignore">If set to true the INSERT INTO query will be INSERT IGNORE INTO</param>
-        public SQLInsert(RowList<T> rows, bool withDelete = true, bool ignore = false)
+        public SQLInsert(RowList<T> rows, bool withDelete = true, bool ignore = false, bool onDuplicateKeyUpdate = false)
         {
             _rows = rows;
             _insertHeader = new SQLInsertHeader<T>(ignore).Build();
             _withDelete = withDelete;
+            _onDuplicateKeyUpdate = onDuplicateKeyUpdate;
+            _onDuplicateKeyUpdateClause = BuildOnDuplicateKeyUpdateClause();
         }
 
         /// <summary>
@@ -447,7 +451,7 @@ public string Build()
             {
                 if (count >= MaxRowsPerInsert)
                 {
-                    query.ReplaceLastCommaWithSemicolon();
+                    FinalizeCurrentInsert(query);
                     query.Append(Environment.NewLine);
                     query.Append(_insertHeader);
                     count = 0;
@@ -456,10 +460,49 @@ public string Build()
                 query.Append(Environment.NewLine);
                 count++;
             }
-            query.ReplaceLastCommaWithSemicolon();
+            FinalizeCurrentInsert(query);
 
             return query.ToString();
         }
+
+        private void FinalizeCurrentInsert(StringBuilder query)
+        {
+            if (_onDuplicateKeyUpdate && !string.IsNullOrEmpty(_onDuplicateKeyUpdateClause))
+            {
+                query.ReplaceLastComma($" {_onDuplicateKeyUpdateClause};");
+                return;
+            }
+
+            query.ReplaceLastCommaWithSemicolon();
+        }
+
+        private static string BuildOnDuplicateKeyUpdateClause()
+        {
+            var fields = SQLUtil.GetFields<T>();
+            var assignments = new List<string>();
+
+            foreach (var field in fields)
+            {
+                var firstAttribute = field.Item3.First();
+                if (firstAttribute.IsPrimaryKey)
+                    continue;
+
+                if (field.Item2.FieldType.IsArray)
+                {
+                    for (var i = 0; i < firstAttribute.Count; ++i)
+                    {
+                        var fieldName = SQLUtil.AddBackQuotes(firstAttribute.Name + (firstAttribute.StartAtZero ? i : i + 1));
+                        assignments.Add($"{fieldName}=VALUES({fieldName})");
+                    }
+
+                    continue;
+                }
+
+                assignments.Add($"{field.Item1}=VALUES({field.Item1})");
+            }
+
+            return assignments.Count == 0 ? string.Empty : "ON DUPLICATE KEY UPDATE " + string.Join(SQLUtil.CommaSeparator, assignments);
+        }
     }
 
     internal class SQLInsertHeader<T> : ISQLQuery where T : IDataModel

WowPacketParser/SQL/SQLUtil.cs

@@ -65,33 +65,78 @@ public static string EscapeString(string str)
         }
 
         /// <summary>
-        /// Replaces the last comma with semicolon in StringBuilder
+        /// Replaces the last comma with a custom suffix in StringBuilder.
+        /// Walks backward until it finds the last non-comment line.
         /// </summary>
-        /// <param name="str"></param>
-        /// <returns></returns>
-        public static void ReplaceLastCommaWithSemicolon(this StringBuilder str)
+        public static void ReplaceLastComma(this StringBuilder str, string replacement)
         {
-            bool isComment = false;
-            int lastCommaPos = -1;
-            for (var i = str.Length - 1; i > 0; i--)
+            for (var lineEnd = str.Length - 1; lineEnd >= 0;)
             {
-                if (i >= 3 && str[i - 3] == ',' && str[i - 2] == ' ' && str[i - 1] == '-' && str[i] == '-')
+                while (lineEnd >= 0 && (str[lineEnd] == '\r' || str[lineEnd] == '\n'))
+                    --lineEnd;
+
+                if (lineEnd < 0)
+                    return;
+
+                var lineStart = lineEnd;
+                while (lineStart > 0 && str[lineStart - 1] != '\r' && str[lineStart - 1] != '\n')
+                    --lineStart;
+
+                var contentStart = lineStart;
+                while (contentStart <= lineEnd && char.IsWhiteSpace(str[contentStart]))
+                    ++contentStart;
+
+                if (contentStart > lineEnd)
                 {
-                    str[i - 3] = ';';
-                    isComment = true;
-                    break;
+                    lineEnd = lineStart - 1;
+                    continue;
+                }
+
+                if (contentStart + 1 <= lineEnd && str[contentStart] == '-' && str[contentStart + 1] == '-')
+                {
+                    lineEnd = lineStart - 1;
+                    continue;
+                }
+
+                var commentStart = -1;
+                for (var i = contentStart; i < lineEnd; ++i)
+                {
+                    if (str[i] == '-' && str[i + 1] == '-')
+                    {
+                        commentStart = i;
+                        break;
+                    }
                 }
 
-                if (lastCommaPos == -1 && str[i] == ',')
-                    lastCommaPos = i;
+                var contentEnd = commentStart == -1 ? lineEnd : commentStart - 1;
+                while (contentEnd >= contentStart && char.IsWhiteSpace(str[contentEnd]))
+                    --contentEnd;
+
+                for (var i = contentEnd; i >= contentStart; --i)
+                {
+                    if (str[i] == ';')
+                        return;
+
+                    if (str[i] == ',')
+                    {
+                        str.Remove(i, 1);
+                        str.Insert(i, replacement);
+                        return;
+                    }
+                }
 
-                // only interact with last line, skip trailing newline
-                if ((str[i] == '\n' || str[i] == '\r') && i < str.Length - 3)
-                    break;
+                lineEnd = lineStart - 1;
             }
+        }
 
-            if (!isComment && lastCommaPos != -1)
-                str[lastCommaPos] = ';';
+        /// <summary>
+        /// Replaces the last comma with semicolon in StringBuilder
+        /// </summary>
+        /// <param name="str"></param>
+        /// <returns></returns>
+        public static void ReplaceLastCommaWithSemicolon(this StringBuilder str)
+        {
+            str.ReplaceLastComma(";");
         }
 
         /// <summary>
@@ -208,14 +253,15 @@ public static bool IsHotfixTable<T>() where T : IDataModel
         /// <param name="storeList"><see cref="DataBag{T}"/> with items form sniff.</param>
         /// <param name="dbList"><see cref="DataBag{T}"/> with items from database.</param>
         /// <param name="storeType">Are we dealing with Spells, Quests, Units, ...?</param>
-        public static string Compare<T>(IEnumerable<Tuple<T, TimeSpan?>> storeList, RowList<T> dbList, StoreNameType storeType)
+        public static string Compare<T>(IEnumerable<Tuple<T, TimeSpan?>> storeList, RowList<T> dbList, StoreNameType storeType, bool useOnDuplicateKeyUpdateForInserts = false)
             where T : IDataModel, new()
         {
             var primaryKey = GetFirstPrimaryKey<T>();
             return Compare(storeList, dbList,
                 t => storeType != StoreNameType.None
                     ? StoreGetters.GetName(storeType, Convert.ToInt32(primaryKey.GetValue(t)), false)
-                    : "");
+                    : "",
+                useOnDuplicateKeyUpdateForInserts);
         }
 
         /// <summary>
@@ -229,7 +275,7 @@ public static string Compare<T>(IEnumerable<Tuple<T, TimeSpan?>> storeList, RowL
         /// <param name="dbList">Dictionary retrieved from  DB</param>
         /// <param name="commentSetter"></param>
         /// <returns>A string containing full SQL queries</returns>
-        public static string Compare<T>(IEnumerable<Tuple<T, TimeSpan?>> storeList, RowList<T> dbList, Func<T, string> commentSetter)
+        public static string Compare<T>(IEnumerable<Tuple<T, TimeSpan?>> storeList, RowList<T> dbList, Func<T, string> commentSetter, bool useOnDuplicateKeyUpdateForInserts = false)
             where T : IDataModel, new()
         {
             if (!IsTableVisible<T>())
@@ -334,7 +380,7 @@ public static string Compare<T>(IEnumerable<Tuple<T, TimeSpan?>> storeList, RowL
                 }
             }
 
-            return new SQLInsert<T>(rowsIns).Build() + Environment.NewLine +
+            return new SQLInsert<T>(rowsIns, withDelete: !useOnDuplicateKeyUpdateForInserts, onDuplicateKeyUpdate: useOnDuplicateKeyUpdateForInserts).Build() + Environment.NewLine +
                    new SQLUpdate<T>(rowsUpd).Build();
         }