Overwrite

Complete Overwrite of the Folder with the free shard. ServUO 57.3 has been added.
This commit is contained in:
Unstable Kitsune
2023-11-28 23:20:26 -05:00
parent 3cd54811de
commit b918192e4e
11608 changed files with 2644205 additions and 47 deletions

View File

@@ -0,0 +1,96 @@
/***************************************************************************
* BinaryMemoryWriter.cs
* -------------------
* begin : May 1, 2002
* copyright : (C) The RunUO Software Team
* email : info@runuo.com
*
* $Id: BinaryMemoryWriter.cs 37 2006-06-19 17:28:24Z mark $
*
***************************************************************************/
/***************************************************************************
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
***************************************************************************/
using System;
using System.IO;
namespace Server
{
public sealed class BinaryMemoryWriter : BinaryFileWriter
{
private static byte[] indexBuffer;
private readonly MemoryStream stream;
public BinaryMemoryWriter()
: base(new MemoryStream(512), true)
{
this.stream = this.UnderlyingStream as MemoryStream;
}
protected override int BufferSize
{
get
{
return 512;
}
}
public int CommitTo(SequentialFileWriter dataFile, SequentialFileWriter indexFile, int typeCode, int serial)
{
this.Flush();
byte[] buffer = this.stream.GetBuffer();
int length = (int)this.stream.Length;
long position = dataFile.Position;
dataFile.Write(buffer, 0, length);
if (indexBuffer == null)
{
indexBuffer = new byte[20];
}
indexBuffer[0] = (byte)(typeCode);
indexBuffer[1] = (byte)(typeCode >> 8);
indexBuffer[2] = (byte)(typeCode >> 16);
indexBuffer[3] = (byte)(typeCode >> 24);
indexBuffer[4] = (byte)(serial);
indexBuffer[5] = (byte)(serial >> 8);
indexBuffer[6] = (byte)(serial >> 16);
indexBuffer[7] = (byte)(serial >> 24);
indexBuffer[8] = (byte)(position);
indexBuffer[9] = (byte)(position >> 8);
indexBuffer[10] = (byte)(position >> 16);
indexBuffer[11] = (byte)(position >> 24);
indexBuffer[12] = (byte)(position >> 32);
indexBuffer[13] = (byte)(position >> 40);
indexBuffer[14] = (byte)(position >> 48);
indexBuffer[15] = (byte)(position >> 56);
indexBuffer[16] = (byte)(length);
indexBuffer[17] = (byte)(length >> 8);
indexBuffer[18] = (byte)(length >> 16);
indexBuffer[19] = (byte)(length >> 24);
indexFile.Write(indexBuffer, 0, indexBuffer.Length);
this.stream.SetLength(0);
return length;
}
}
}

View File

@@ -0,0 +1,41 @@
using System;
using System.Threading;
namespace Server
{
public sealed class DualSaveStrategy : StandardSaveStrategy
{
public DualSaveStrategy()
{
}
public override string Name
{
get
{
return "Dual";
}
}
public override void Save(SaveMetrics metrics, bool permitBackgroundWrite)
{
this.PermitBackgroundWrite = permitBackgroundWrite;
Thread saveThread = new Thread(delegate()
{
this.SaveItems(metrics);
});
saveThread.Name = "Item Save Subset";
saveThread.Start();
this.SaveMobiles(metrics);
this.SaveGuilds(metrics);
this.SaveData(metrics);
saveThread.Join();
if (permitBackgroundWrite && this.UseSequentialWriters) //If we're permitted to write in the background, but we don't anyways, then notify.
World.NotifyDiskWriteComplete();
}
}
}

View File

@@ -0,0 +1,333 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using CustomsFramework;
using Server.Guilds;
namespace Server
{
public sealed class DynamicSaveStrategy : SaveStrategy
{
private readonly ConcurrentBag<Item> _decayBag;
private readonly BlockingCollection<QueuedMemoryWriter> _itemThreadWriters;
private readonly BlockingCollection<QueuedMemoryWriter> _mobileThreadWriters;
private readonly BlockingCollection<QueuedMemoryWriter> _guildThreadWriters;
private readonly BlockingCollection<QueuedMemoryWriter> _dataThreadWriters;
private SaveMetrics _metrics;
private SequentialFileWriter _itemData, _itemIndex;
private SequentialFileWriter _mobileData, _mobileIndex;
private SequentialFileWriter _guildData, _guildIndex;
private SequentialFileWriter _customData, _customIndex;
public DynamicSaveStrategy()
{
this._decayBag = new ConcurrentBag<Item>();
this._itemThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
this._mobileThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
this._guildThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
this._dataThreadWriters = new BlockingCollection<QueuedMemoryWriter>();
}
public override string Name
{
get
{
return "Dynamic";
}
}
public override void Save(SaveMetrics metrics, bool permitBackgroundWrite)
{
this._metrics = metrics;
this.OpenFiles();
Task[] saveTasks = new Task[4];
saveTasks[0] = this.SaveItems();
saveTasks[1] = this.SaveMobiles();
saveTasks[2] = this.SaveGuilds();
saveTasks[3] = this.SaveData();
this.SaveTypeDatabases();
if (permitBackgroundWrite)
{
//This option makes it finish the writing to disk in the background, continuing even after Save() returns.
Task.Factory.ContinueWhenAll(saveTasks, _ =>
{
this.CloseFiles();
World.NotifyDiskWriteComplete();
});
}
else
{
Task.WaitAll(saveTasks); //Waits for the completion of all of the tasks(committing to disk)
this.CloseFiles();
}
}
public override void ProcessDecay()
{
Item item;
while (this._decayBag.TryTake(out item))
{
if (item.OnDecay())
{
item.Delete();
}
}
}
private Task StartCommitTask(BlockingCollection<QueuedMemoryWriter> threadWriter, SequentialFileWriter data, SequentialFileWriter index)
{
Task commitTask = Task.Factory.StartNew(() =>
{
while (!(threadWriter.IsCompleted))
{
QueuedMemoryWriter writer;
try
{
writer = threadWriter.Take();
}
catch (InvalidOperationException)
{
//Per MSDN, it's fine if we're here, successful completion of adding can rarely put us into this state.
break;
}
writer.CommitTo(data, index);
}
});
return commitTask;
}
private Task SaveItems()
{
//Start the blocking consumer; this runs in background.
Task commitTask = this.StartCommitTask(this._itemThreadWriters, this._itemData, this._itemIndex);
IEnumerable<Item> items = World.Items.Values;
//Start the producer.
Parallel.ForEach(items, () => new QueuedMemoryWriter(),
(Item item, ParallelLoopState state, QueuedMemoryWriter writer) =>
{
long startPosition = writer.Position;
item.Serialize(writer);
int size = (int)(writer.Position - startPosition);
writer.QueueForIndex(item, size);
if (item.Decays && item.Parent == null && item.Map != Map.Internal && DateTime.UtcNow > (item.LastMoved + item.DecayTime))
{
this._decayBag.Add(item);
}
if (this._metrics != null)
{
this._metrics.OnItemSaved(size);
}
return writer;
},
(writer) =>
{
writer.Flush();
this._itemThreadWriters.Add(writer);
});
this._itemThreadWriters.CompleteAdding(); //We only get here after the Parallel.ForEach completes. Lets our task
return commitTask;
}
private Task SaveMobiles()
{
//Start the blocking consumer; this runs in background.
Task commitTask = this.StartCommitTask(this._mobileThreadWriters, this._mobileData, this._mobileIndex);
IEnumerable<Mobile> mobiles = World.Mobiles.Values;
//Start the producer.
Parallel.ForEach(mobiles, () => new QueuedMemoryWriter(),
(Mobile mobile, ParallelLoopState state, QueuedMemoryWriter writer) =>
{
long startPosition = writer.Position;
mobile.Serialize(writer);
int size = (int)(writer.Position - startPosition);
writer.QueueForIndex(mobile, size);
if (this._metrics != null)
{
this._metrics.OnMobileSaved(size);
}
return writer;
},
(writer) =>
{
writer.Flush();
this._mobileThreadWriters.Add(writer);
});
this._mobileThreadWriters.CompleteAdding(); //We only get here after the Parallel.ForEach completes. Lets our task tell the consumer that we're done
return commitTask;
}
private Task SaveGuilds()
{
//Start the blocking consumer; this runs in background.
Task commitTask = this.StartCommitTask(this._guildThreadWriters, this._guildData, this._guildIndex);
IEnumerable<BaseGuild> guilds = BaseGuild.List.Values;
//Start the producer.
Parallel.ForEach(guilds, () => new QueuedMemoryWriter(),
(BaseGuild guild, ParallelLoopState state, QueuedMemoryWriter writer) =>
{
long startPosition = writer.Position;
guild.Serialize(writer);
int size = (int)(writer.Position - startPosition);
writer.QueueForIndex(guild, size);
if (this._metrics != null)
{
this._metrics.OnGuildSaved(size);
}
return writer;
},
(writer) =>
{
writer.Flush();
this._guildThreadWriters.Add(writer);
});
this._guildThreadWriters.CompleteAdding(); //We only get here after the Parallel.ForEach completes. Lets our task
return commitTask;
}
private Task SaveData()
{
Task commitTask = this.StartCommitTask(this._dataThreadWriters, this._customData, this._customIndex);
IEnumerable<SaveData> data = World.Data.Values;
Parallel.ForEach(data, () => new QueuedMemoryWriter(),
(SaveData saveData, ParallelLoopState state, QueuedMemoryWriter writer) =>
{
long startPosition = writer.Position;
saveData.Serialize(writer);
int size = (int)(writer.Position - startPosition);
writer.QueueForIndex(saveData, size);
if (this._metrics != null)
this._metrics.OnDataSaved(size);
return writer;
},
(writer) =>
{
writer.Flush();
this._dataThreadWriters.Add(writer);
});
this._dataThreadWriters.CompleteAdding();
return commitTask;
}
private void OpenFiles()
{
this._itemData = new SequentialFileWriter(World.ItemDataPath, this._metrics);
this._itemIndex = new SequentialFileWriter(World.ItemIndexPath, this._metrics);
this._mobileData = new SequentialFileWriter(World.MobileDataPath, this._metrics);
this._mobileIndex = new SequentialFileWriter(World.MobileIndexPath, this._metrics);
this._guildData = new SequentialFileWriter(World.GuildDataPath, this._metrics);
this._guildIndex = new SequentialFileWriter(World.GuildIndexPath, this._metrics);
this._customData = new SequentialFileWriter(World.DataBinaryPath, this._metrics);
this._customIndex = new SequentialFileWriter(World.DataIndexPath, this._metrics);
this.WriteCount(this._itemIndex, World.Items.Count);
this.WriteCount(this._mobileIndex, World.Mobiles.Count);
this.WriteCount(this._guildIndex, BaseGuild.List.Count);
this.WriteCount(this._customIndex, World.Data.Count);
}
private void CloseFiles()
{
this._itemData.Close();
this._itemIndex.Close();
this._mobileData.Close();
this._mobileIndex.Close();
this._guildData.Close();
this._guildIndex.Close();
this._customData.Close();
this._customIndex.Close();
}
private void WriteCount(SequentialFileWriter indexFile, int count)
{
//Equiv to GenericWriter.Write( (int)count );
byte[] buffer = new byte[4];
buffer[0] = (byte)(count);
buffer[1] = (byte)(count >> 8);
buffer[2] = (byte)(count >> 16);
buffer[3] = (byte)(count >> 24);
indexFile.Write(buffer, 0, buffer.Length);
}
private void SaveTypeDatabases()
{
this.SaveTypeDatabase(World.ItemTypesPath, World.m_ItemTypes);
this.SaveTypeDatabase(World.MobileTypesPath, World.m_MobileTypes);
this.SaveTypeDatabase(World.DataTypesPath, World._DataTypes);
}
private void SaveTypeDatabase(string path, List<Type> types)
{
BinaryFileWriter bfw = new BinaryFileWriter(path, false);
bfw.Write(types.Count);
foreach (Type type in types)
{
bfw.Write(type.FullName);
}
bfw.Flush();
bfw.Close();
}
}
}

View File

@@ -0,0 +1,134 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using Microsoft.Win32.SafeHandles;
namespace Server
{
public static class FileOperations
{
public const int KB = 1024;
public const int MB = 1024 * KB;
private const FileOptions NoBuffering = (FileOptions)0x20000000;
[DllImport("Kernel32", CharSet = CharSet.Auto, SetLastError = true)]
private static extern SafeFileHandle CreateFile(string lpFileName, int dwDesiredAccess, FileShare dwShareMode, IntPtr securityAttrs, FileMode dwCreationDisposition, int dwFlagsAndAttributes, IntPtr hTemplateFile);
private static int bufferSize = 1 * MB;
private static int concurrency = 1;
private static bool unbuffered = true;
public static int BufferSize
{
get
{
return bufferSize;
}
set
{
bufferSize = value;
}
}
public static int Concurrency
{
get
{
return concurrency;
}
set
{
concurrency = value;
}
}
public static bool Unbuffered
{
get
{
return unbuffered;
}
set
{
unbuffered = value;
}
}
public static bool AreSynchronous
{
get
{
return (concurrency < 1);
}
}
public static bool AreAsynchronous
{
get
{
return (concurrency > 0);
}
}
public static FileStream OpenSequentialStream(string path, FileMode mode, FileAccess access, FileShare share)
{
FileOptions options = FileOptions.SequentialScan;
if (concurrency > 0)
{
options |= FileOptions.Asynchronous;
}
if (unbuffered)
{
options |= NoBuffering;
}
else
{
return new FileStream(path, mode, access, share, bufferSize, options);
}
SafeFileHandle fileHandle = CreateFile(path, (int)access, share, IntPtr.Zero, mode, (int)options, IntPtr.Zero);
if (fileHandle.IsInvalid)
{
throw new IOException();
}
return new UnbufferedFileStream(fileHandle, access, bufferSize, (concurrency > 0));
}
private class UnbufferedFileStream : FileStream
{
private readonly SafeFileHandle fileHandle;
public UnbufferedFileStream(SafeFileHandle fileHandle, FileAccess access, int bufferSize, bool isAsync)
: base(fileHandle, access, bufferSize, isAsync)
{
this.fileHandle = fileHandle;
}
public override void Write(byte[] array, int offset, int count)
{
base.Write(array, offset, bufferSize);
}
public override IAsyncResult BeginWrite(byte[] array, int offset, int numBytes, AsyncCallback userCallback, object stateObject)
{
return base.BeginWrite(array, offset, bufferSize, userCallback, stateObject);
}
protected override void Dispose(bool disposing)
{
if (!this.fileHandle.IsClosed)
{
this.fileHandle.Close();
}
base.Dispose(disposing);
}
}
}
}

View File

@@ -0,0 +1,263 @@
using System;
using System.Collections.Generic;
using System.Threading;
using Server.Network;
namespace Server
{
public delegate void FileCommitCallback(FileQueue.Chunk chunk);
public sealed class FileQueue : IDisposable
{
private static int bufferSize;
private static BufferPool bufferPool;
private readonly object syncRoot;
private readonly Chunk[] active;
private readonly Queue<Page> pending;
private Page buffered;
private readonly FileCommitCallback callback;
private int activeCount;
private ManualResetEvent idle;
private long position;
public FileQueue(int concurrentWrites, FileCommitCallback callback)
{
if (concurrentWrites < 1)
{
throw new ArgumentOutOfRangeException("concurrentWrites");
}
else if (bufferSize < 1)
{
throw new ArgumentOutOfRangeException("bufferSize");
}
else if (callback == null)
{
throw new ArgumentNullException("callback");
}
this.syncRoot = new object();
this.active = new Chunk[concurrentWrites];
this.pending = new Queue<Page>();
this.callback = callback;
this.idle = new ManualResetEvent(true);
}
static FileQueue()
{
bufferSize = FileOperations.BufferSize;
bufferPool = new BufferPool("File Buffers", 64, bufferSize);
}
public long Position
{
get
{
return this.position;
}
}
public void Dispose()
{
if (this.idle != null)
{
this.idle.Close();
this.idle = null;
}
}
public void Flush()
{
if (this.buffered.buffer != null)
{
this.Append(this.buffered);
this.buffered.buffer = null;
this.buffered.length = 0;
}
/*lock ( syncRoot ) {
if ( pending.Count > 0 ) {
idle.Reset();
}
for ( int slot = 0; slot < active.Length && pending.Count > 0; ++slot ) {
if ( active[slot] == null ) {
Page page = pending.Dequeue();
active[slot] = new Chunk( this, slot, page.buffer, 0, page.length );
++activeCount;
callback( active[slot] );
}
}
}*/
this.idle.WaitOne();
}
public void Enqueue(byte[] buffer, int offset, int size)
{
if (buffer == null)
{
throw new ArgumentNullException("buffer");
}
else if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset");
}
else if (size < 0)
{
throw new ArgumentOutOfRangeException("size");
}
else if ((buffer.Length - offset) < size)
{
throw new ArgumentException();
}
this.position += size;
while (size > 0)
{
if (this.buffered.buffer == null)
{ // nothing yet buffered
this.buffered.buffer = bufferPool.AcquireBuffer();
}
byte[] page = this.buffered.buffer; // buffer page
int pageSpace = page.Length - this.buffered.length; // available bytes in page
int byteCount = (size > pageSpace ? pageSpace : size); // how many bytes we can copy over
Buffer.BlockCopy(buffer, offset, page, this.buffered.length, byteCount);
this.buffered.length += byteCount;
offset += byteCount;
size -= byteCount;
if (this.buffered.length == page.Length)
{ // page full
this.Append(this.buffered);
this.buffered.buffer = null;
this.buffered.length = 0;
}
}
}
private void Append(Page page)
{
lock (this.syncRoot)
{
if (this.activeCount == 0)
{
this.idle.Reset();
}
++this.activeCount;
for (int slot = 0; slot < this.active.Length; ++slot)
{
if (this.active[slot] == null)
{
this.active[slot] = new Chunk(this, slot, page.buffer, 0, page.length);
this.callback(this.active[slot]);
return;
}
}
this.pending.Enqueue(page);
}
}
private void Commit(Chunk chunk, int slot)
{
if (slot < 0 || slot >= this.active.Length)
{
throw new ArgumentOutOfRangeException("slot");
}
lock (this.syncRoot)
{
if (this.active[slot] != chunk)
{
throw new ArgumentException();
}
bufferPool.ReleaseBuffer(chunk.Buffer);
if (this.pending.Count > 0)
{
Page page = this.pending.Dequeue();
this.active[slot] = new Chunk(this, slot, page.buffer, 0, page.length);
this.callback(this.active[slot]);
}
else
{
this.active[slot] = null;
}
--this.activeCount;
if (this.activeCount == 0)
{
this.idle.Set();
}
}
}
private struct Page
{
public byte[] buffer;
public int length;
}
public sealed class Chunk
{
private readonly FileQueue owner;
private readonly int slot;
private readonly byte[] buffer;
private readonly int offset;
private readonly int size;
public Chunk(FileQueue owner, int slot, byte[] buffer, int offset, int size)
{
this.owner = owner;
this.slot = slot;
this.buffer = buffer;
this.offset = offset;
this.size = size;
}
public byte[] Buffer
{
get
{
return this.buffer;
}
}
public int Offset
{
get
{
return 0;
}
}
public int Size
{
get
{
return this.size;
}
}
public void Commit()
{
this.owner.Commit(this, this.slot);
}
}
}
}

View File

@@ -0,0 +1,385 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Threading;
using CustomsFramework;
using Server.Guilds;
namespace Server
{
public sealed class ParallelSaveStrategy : SaveStrategy
{
private readonly int processorCount;
private readonly Queue<Item> _decayQueue;
private SaveMetrics metrics;
private SequentialFileWriter itemData, itemIndex;
private SequentialFileWriter mobileData, mobileIndex;
private SequentialFileWriter guildData, guildIndex;
private SequentialFileWriter customData, customIndex;
private Consumer[] consumers;
private int cycle;
private bool finished;
public ParallelSaveStrategy(int processorCount)
{
this.processorCount = processorCount;
this._decayQueue = new Queue<Item>();
}
public override string Name
{
get
{
return "Parallel";
}
}
public override void Save(SaveMetrics metrics, bool permitBackgroundWrite)
{
this.metrics = metrics;
this.OpenFiles();
this.consumers = new Consumer[this.GetThreadCount()];
for (int i = 0; i < this.consumers.Length; ++i)
{
this.consumers[i] = new Consumer(this, 256);
}
IEnumerable<ISerializable> collection = new Producer();
foreach (ISerializable value in collection)
{
while (!this.Enqueue(value))
{
if (!this.Commit())
{
Thread.Sleep(0);
}
}
}
this.finished = true;
this.SaveTypeDatabases();
WaitHandle.WaitAll(
Array.ConvertAll<Consumer, WaitHandle>(
this.consumers,
delegate(Consumer input)
{
return input.completionEvent;
}));
this.Commit();
this.CloseFiles();
}
public override void ProcessDecay()
{
while (this._decayQueue.Count > 0)
{
Item item = this._decayQueue.Dequeue();
if (item.OnDecay())
{
item.Delete();
}
}
}
private int GetThreadCount()
{
return this.processorCount - 1;
}
private void SaveTypeDatabases()
{
this.SaveTypeDatabase(World.ItemTypesPath, World.m_ItemTypes);
this.SaveTypeDatabase(World.MobileTypesPath, World.m_MobileTypes);
this.SaveTypeDatabase(World.DataTypesPath, World._DataTypes);
}
private void SaveTypeDatabase(string path, List<Type> types)
{
BinaryFileWriter bfw = new BinaryFileWriter(path, false);
bfw.Write(types.Count);
foreach (Type type in types)
{
bfw.Write(type.FullName);
}
bfw.Flush();
bfw.Close();
}
private void OpenFiles()
{
this.itemData = new SequentialFileWriter(World.ItemDataPath, this.metrics);
this.itemIndex = new SequentialFileWriter(World.ItemIndexPath, this.metrics);
this.mobileData = new SequentialFileWriter(World.MobileDataPath, this.metrics);
this.mobileIndex = new SequentialFileWriter(World.MobileIndexPath, this.metrics);
this.guildData = new SequentialFileWriter(World.GuildDataPath, this.metrics);
this.guildIndex = new SequentialFileWriter(World.GuildIndexPath, this.metrics);
this.customData = new SequentialFileWriter(World.DataBinaryPath, this.metrics);
this.customIndex = new SequentialFileWriter(World.DataIndexPath, this.metrics);
this.WriteCount(this.itemIndex, World.Items.Count);
this.WriteCount(this.mobileIndex, World.Mobiles.Count);
this.WriteCount(this.guildIndex, BaseGuild.List.Count);
this.WriteCount(this.customIndex, World.Data.Count);
}
private void WriteCount(SequentialFileWriter indexFile, int count)
{
byte[] buffer = new byte[4];
buffer[0] = (byte)(count);
buffer[1] = (byte)(count >> 8);
buffer[2] = (byte)(count >> 16);
buffer[3] = (byte)(count >> 24);
indexFile.Write(buffer, 0, buffer.Length);
}
private void CloseFiles()
{
this.itemData.Close();
this.itemIndex.Close();
this.mobileData.Close();
this.mobileIndex.Close();
this.guildData.Close();
this.guildIndex.Close();
this.customData.Close();
this.customIndex.Close();
World.NotifyDiskWriteComplete();
}
private void OnSerialized(ConsumableEntry entry)
{
ISerializable value = entry.value;
BinaryMemoryWriter writer = entry.writer;
Item item = value as Item;
if (item != null)
this.Save(item, writer);
else
{
Mobile mob = value as Mobile;
if (mob != null)
this.Save(mob, writer);
else
{
BaseGuild guild = value as BaseGuild;
if (guild != null)
this.Save(guild, writer);
else
{
SaveData data = value as SaveData;
if (data != null)
this.Save(data, writer);
}
}
}
}
private void Save(Item item, BinaryMemoryWriter writer)
{
int length = writer.CommitTo(this.itemData, this.itemIndex, item.m_TypeRef, item.Serial);
if (this.metrics != null)
{
this.metrics.OnItemSaved(length);
}
if (item.Decays && item.Parent == null && item.Map != Map.Internal && DateTime.UtcNow > (item.LastMoved + item.DecayTime))
{
this._decayQueue.Enqueue(item);
}
}
private void Save(Mobile mob, BinaryMemoryWriter writer)
{
int length = writer.CommitTo(this.mobileData, this.mobileIndex, mob.m_TypeRef, mob.Serial);
if (this.metrics != null)
{
this.metrics.OnMobileSaved(length);
}
}
private void Save(BaseGuild guild, BinaryMemoryWriter writer)
{
int length = writer.CommitTo(this.guildData, this.guildIndex, 0, guild.Id);
if (this.metrics != null)
{
this.metrics.OnGuildSaved(length);
}
}
private void Save(SaveData data, BinaryMemoryWriter writer)
{
int length = writer.CommitTo(this.customData, this.customIndex, data._TypeID, data.Serial);
if (this.metrics != null)
this.metrics.OnDataSaved(length);
}
private bool Enqueue(ISerializable value)
{
for (int i = 0; i < this.consumers.Length; ++i)
{
Consumer consumer = this.consumers[this.cycle++ % this.consumers.Length];
if ((consumer.tail - consumer.head) < consumer.buffer.Length)
{
consumer.buffer[consumer.tail % consumer.buffer.Length].value = value;
consumer.tail++;
return true;
}
}
return false;
}
private bool Commit()
{
bool committed = false;
for (int i = 0; i < this.consumers.Length; ++i)
{
Consumer consumer = this.consumers[i];
while (consumer.head < consumer.done)
{
this.OnSerialized(consumer.buffer[consumer.head % consumer.buffer.Length]);
consumer.head++;
committed = true;
}
}
return committed;
}
private struct ConsumableEntry
{
public ISerializable value;
public BinaryMemoryWriter writer;
}
private sealed class Producer : IEnumerable<ISerializable>
{
private readonly IEnumerable<Item> items;
private readonly IEnumerable<Mobile> mobiles;
private readonly IEnumerable<BaseGuild> guilds;
private readonly IEnumerable<SaveData> data;
public Producer()
{
this.items = World.Items.Values;
this.mobiles = World.Mobiles.Values;
this.guilds = BaseGuild.List.Values;
this.data = World.Data.Values;
}
public IEnumerator<ISerializable> GetEnumerator()
{
foreach (Item item in this.items)
yield return item;
foreach (Mobile mob in this.mobiles)
yield return mob;
foreach (BaseGuild guild in this.guilds)
yield return guild;
foreach (SaveData data in this.data)
yield return data;
}
IEnumerator IEnumerable.GetEnumerator()
{
throw new NotImplementedException();
}
}
private sealed class Consumer
{
public readonly ManualResetEvent completionEvent;
public readonly ConsumableEntry[] buffer;
public int head, done, tail;
private readonly ParallelSaveStrategy owner;
private readonly Thread thread;
public Consumer(ParallelSaveStrategy owner, int bufferSize)
{
this.owner = owner;
this.buffer = new ConsumableEntry[bufferSize];
for (int i = 0; i < this.buffer.Length; ++i)
{
this.buffer[i].writer = new BinaryMemoryWriter();
}
this.completionEvent = new ManualResetEvent(false);
this.thread = new Thread(Processor);
this.thread.Name = "Parallel Serialization Thread";
this.thread.Start();
}
private void Processor()
{
try
{
while (!this.owner.finished)
{
this.Process();
Thread.Sleep(0);
}
this.Process();
this.completionEvent.Set();
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
private void Process()
{
ConsumableEntry entry;
while (this.done < this.tail)
{
entry = this.buffer[this.done % this.buffer.Length];
entry.value.Serialize(entry.writer);
++this.done;
}
}
}
}
}

View File

@@ -0,0 +1,119 @@
#region References
using System;
using System.IO;
#endregion
namespace Server
{
public static class Persistence
{
public static void Serialize(string path, Action<GenericWriter> serializer)
{
Serialize(new FileInfo(path), serializer);
}
public static void Serialize(FileInfo file, Action<GenericWriter> serializer)
{
file.Refresh();
if (file.Directory != null && !file.Directory.Exists)
{
file.Directory.Create();
}
if (!file.Exists)
{
file.Create().Close();
}
file.Refresh();
using (var fs = file.OpenWrite())
{
var writer = new BinaryFileWriter(fs, true);
try
{
serializer(writer);
}
finally
{
writer.Flush();
writer.Close();
}
}
}
public static void Deserialize(string path, Action<GenericReader> deserializer)
{
Deserialize(path, deserializer, true);
}
public static void Deserialize(FileInfo file, Action<GenericReader> deserializer)
{
Deserialize(file, deserializer, true);
}
public static void Deserialize(string path, Action<GenericReader> deserializer, bool ensure)
{
Deserialize(new FileInfo(path), deserializer, ensure);
}
public static void Deserialize(FileInfo file, Action<GenericReader> deserializer, bool ensure)
{
file.Refresh();
if (file.Directory != null && !file.Directory.Exists)
{
if (!ensure)
{
throw new DirectoryNotFoundException();
}
file.Directory.Create();
}
if (!file.Exists)
{
if (!ensure)
{
throw new FileNotFoundException
{
Source = file.FullName
};
}
file.Create().Close();
}
file.Refresh();
using (var fs = file.OpenRead())
{
var reader = new BinaryFileReader(new BinaryReader(fs));
try
{
deserializer(reader);
}
catch (EndOfStreamException eos)
{
if (file.Length > 0)
{
throw new Exception(String.Format("[Persistance]: {0}", eos));
}
}
catch (Exception e)
{
Utility.WriteConsoleColor(ConsoleColor.Red, "[Persistance]: An error was encountered while loading a saved object");
throw new Exception(String.Format("[Persistance]: {0}", e));
}
finally
{
reader.Close();
}
}
}
}
}

View File

@@ -0,0 +1,131 @@
/***************************************************************************
* QueuedMemoryWriter.cs
* -------------------
* begin : December 16, 2010
* copyright : (C) The RunUO Software Team
* email : info@runuo.com
*
* $Id: QueuedMemoryWriter.cs 645 2010-12-23 11:36:25Z asayre $
*
***************************************************************************/
/***************************************************************************
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
***************************************************************************/
using System;
using System.Collections.Generic;
using System.IO;
namespace Server
{
public sealed class QueuedMemoryWriter : BinaryFileWriter
{
private readonly MemoryStream _memStream;
private readonly List<IndexInfo> _orderedIndexInfo = new List<IndexInfo>();
public QueuedMemoryWriter()
: base(new MemoryStream(1024 * 1024), true)
{
this._memStream = this.UnderlyingStream as MemoryStream;
}
protected override int BufferSize
{
get
{
return 512;
}
}
public void QueueForIndex(ISerializable serializable, int size)
{
IndexInfo info;
info.size = size;
info.typeCode = serializable.TypeReference; //For guilds, this will automagically be zero.
info.serial = serializable.SerialIdentity;
this._orderedIndexInfo.Add(info);
}
public void CommitTo(SequentialFileWriter dataFile, SequentialFileWriter indexFile)
{
this.Flush();
int memLength = (int)this._memStream.Position;
if (memLength > 0)
{
byte[] memBuffer = this._memStream.GetBuffer();
long actualPosition = dataFile.Position;
dataFile.Write(memBuffer, 0, memLength); //The buffer contains the data from many items.
//Console.WriteLine("Writing {0} bytes starting at {1}, with {2} things", memLength, actualPosition, _orderedIndexInfo.Count);
byte[] indexBuffer = new byte[20];
//int indexWritten = _orderedIndexInfo.Count * indexBuffer.Length;
//int totalWritten = memLength + indexWritten
for (int i = 0; i < this._orderedIndexInfo.Count; i++)
{
IndexInfo info = this._orderedIndexInfo[i];
int typeCode = info.typeCode;
int serial = info.serial;
int length = info.size;
indexBuffer[0] = (byte)(info.typeCode);
indexBuffer[1] = (byte)(info.typeCode >> 8);
indexBuffer[2] = (byte)(info.typeCode >> 16);
indexBuffer[3] = (byte)(info.typeCode >> 24);
indexBuffer[4] = (byte)(info.serial);
indexBuffer[5] = (byte)(info.serial >> 8);
indexBuffer[6] = (byte)(info.serial >> 16);
indexBuffer[7] = (byte)(info.serial >> 24);
indexBuffer[8] = (byte)(actualPosition);
indexBuffer[9] = (byte)(actualPosition >> 8);
indexBuffer[10] = (byte)(actualPosition >> 16);
indexBuffer[11] = (byte)(actualPosition >> 24);
indexBuffer[12] = (byte)(actualPosition >> 32);
indexBuffer[13] = (byte)(actualPosition >> 40);
indexBuffer[14] = (byte)(actualPosition >> 48);
indexBuffer[15] = (byte)(actualPosition >> 56);
indexBuffer[16] = (byte)(info.size);
indexBuffer[17] = (byte)(info.size >> 8);
indexBuffer[18] = (byte)(info.size >> 16);
indexBuffer[19] = (byte)(info.size >> 24);
indexFile.Write(indexBuffer, 0, indexBuffer.Length);
actualPosition += info.size;
}
}
this.Close(); //We're done with this writer.
}
private struct IndexInfo
{
public int size;
public int typeCode;
public int serial;
}
}
}

View File

@@ -0,0 +1,139 @@
using System;
using System.Diagnostics;
namespace Server
{
public sealed class SaveMetrics : IDisposable
{
private const string PerformanceCategoryName = "ServUO";
private const string PerformanceCategoryDesc = "Performance counters for ServUO";
private readonly PerformanceCounter numberOfWorldSaves;
private readonly PerformanceCounter itemsPerSecond;
private readonly PerformanceCounter mobilesPerSecond;
private readonly PerformanceCounter dataPerSecond;
private readonly PerformanceCounter serializedBytesPerSecond;
private readonly PerformanceCounter writtenBytesPerSecond;
public SaveMetrics()
{
if (!PerformanceCounterCategory.Exists(PerformanceCategoryName))
{
CounterCreationDataCollection counters = new CounterCreationDataCollection();
counters.Add(new CounterCreationData(
"Save - Count",
"Number of world saves.",
PerformanceCounterType.NumberOfItems32));
counters.Add(new CounterCreationData(
"Save - Items/sec",
"Number of items saved per second.",
PerformanceCounterType.RateOfCountsPerSecond32));
counters.Add(new CounterCreationData(
"Save - Mobiles/sec",
"Number of mobiles saved per second.",
PerformanceCounterType.RateOfCountsPerSecond32));
counters.Add(new CounterCreationData(
"Save - Customs/sec",
"Number of cores saved per second.",
PerformanceCounterType.RateOfCountsPerSecond32));
counters.Add(new CounterCreationData(
"Save - Serialized bytes/sec",
"Amount of world-save bytes serialized per second.",
PerformanceCounterType.RateOfCountsPerSecond32));
counters.Add(new CounterCreationData(
"Save - Written bytes/sec",
"Amount of world-save bytes written to disk per second.",
PerformanceCounterType.RateOfCountsPerSecond32));
if (!Core.Unix)
{
try
{
PerformanceCounterCategory.Create(PerformanceCategoryName, PerformanceCategoryDesc, PerformanceCounterCategoryType.SingleInstance, counters);
}
catch
{
if (Core.Debug)
Console.WriteLine("Metrics: Metrics enabled. Performance counters creation requires ServUO to be run as Administrator once!");
}
}
else
{
Utility.PushColor(ConsoleColor.Yellow);
Console.WriteLine("WARNING: You've enabled SaveMetrics. This is currently not supported on Unix based operating systems. Please disable this option to hide this message.");
Utility.PopColor();
}
}
this.numberOfWorldSaves = new PerformanceCounter(PerformanceCategoryName, "Save - Count", false);
this.itemsPerSecond = new PerformanceCounter(PerformanceCategoryName, "Save - Items/sec", false);
this.mobilesPerSecond = new PerformanceCounter(PerformanceCategoryName, "Save - Mobiles/sec", false);
this.dataPerSecond = new PerformanceCounter(PerformanceCategoryName, "Save - Customs/sec", false);
this.serializedBytesPerSecond = new PerformanceCounter(PerformanceCategoryName, "Save - Serialized bytes/sec", false);
this.writtenBytesPerSecond = new PerformanceCounter(PerformanceCategoryName, "Save - Written bytes/sec", false);
// increment number of world saves
this.numberOfWorldSaves.Increment();
}
public void OnItemSaved(int numberOfBytes)
{
this.itemsPerSecond.Increment();
this.serializedBytesPerSecond.IncrementBy(numberOfBytes);
}
public void OnMobileSaved(int numberOfBytes)
{
this.mobilesPerSecond.Increment();
this.serializedBytesPerSecond.IncrementBy(numberOfBytes);
}
public void OnGuildSaved(int numberOfBytes)
{
this.serializedBytesPerSecond.IncrementBy(numberOfBytes);
}
public void OnDataSaved(int numberOfBytes)
{
this.dataPerSecond.Increment();
this.serializedBytesPerSecond.IncrementBy(numberOfBytes);
}
public void OnFileWritten(int numberOfBytes)
{
this.writtenBytesPerSecond.IncrementBy(numberOfBytes);
}
private bool isDisposed;
public void Dispose()
{
if (!this.isDisposed)
{
this.isDisposed = true;
this.numberOfWorldSaves.Dispose();
this.itemsPerSecond.Dispose();
this.mobilesPerSecond.Dispose();
this.dataPerSecond.Dispose();
this.serializedBytesPerSecond.Dispose();
this.writtenBytesPerSecond.Dispose();
}
}
}
}

View File

@@ -0,0 +1,40 @@
using System;
namespace Server
{
public abstract class SaveStrategy
{
public abstract string Name { get; }
public static SaveStrategy Acquire()
{
if (Core.MultiProcessor)
{
int processorCount = Core.ProcessorCount;
#if DynamicSaveStrategy
if (processorCount > 2)
{
return new DynamicSaveStrategy();
}
#else
if (processorCount > 16)
{
return new ParallelSaveStrategy(processorCount);
}
#endif
else
{
return new DualSaveStrategy();
}
}
else
{
return new StandardSaveStrategy();
}
}
public abstract void Save(SaveMetrics metrics, bool permitBackgroundWrite);
public abstract void ProcessDecay();
}
}

View File

@@ -0,0 +1,173 @@
/***************************************************************************
* SequentialFileWriter.cs
* -------------------
* begin : May 1, 2002
* copyright : (C) The RunUO Software Team
* email : info@runuo.com
*
* $Id: SequentialFileWriter.cs 4 2006-06-15 04:28:39Z mark $
*
***************************************************************************/
/***************************************************************************
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
***************************************************************************/
using System;
using System.IO;
namespace Server
{
public sealed class SequentialFileWriter : Stream
{
private readonly SaveMetrics metrics;
private FileStream fileStream;
private FileQueue fileQueue;
private AsyncCallback writeCallback;
public SequentialFileWriter(string path, SaveMetrics metrics)
{
if (path == null)
{
throw new ArgumentNullException("path");
}
this.metrics = metrics;
this.fileStream = FileOperations.OpenSequentialStream(path, FileMode.Create, FileAccess.Write, FileShare.None);
this.fileQueue = new FileQueue(
Math.Max(1, FileOperations.Concurrency),
FileCallback);
}
public override long Position
{
get
{
return this.fileQueue.Position;
}
set
{
throw new InvalidOperationException();
}
}
public override bool CanRead
{
get
{
return false;
}
}
public override bool CanSeek
{
get
{
return false;
}
}
public override bool CanWrite
{
get
{
return true;
}
}
public override long Length
{
get
{
return this.Position;
}
}
public override void Write(byte[] buffer, int offset, int size)
{
this.fileQueue.Enqueue(buffer, offset, size);
}
public override void Flush()
{
this.fileQueue.Flush();
this.fileStream.Flush();
}
public override int Read(byte[] buffer, int offset, int count)
{
throw new InvalidOperationException();
}
public override long Seek(long offset, SeekOrigin origin)
{
throw new InvalidOperationException();
}
public override void SetLength(long value)
{
this.fileStream.SetLength(value);
}
protected override void Dispose(bool disposing)
{
if (this.fileStream != null)
{
this.Flush();
this.fileQueue.Dispose();
this.fileQueue = null;
this.fileStream.Close();
this.fileStream = null;
}
base.Dispose(disposing);
}
private void FileCallback(FileQueue.Chunk chunk)
{
if (FileOperations.AreSynchronous)
{
this.fileStream.Write(chunk.Buffer, chunk.Offset, chunk.Size);
if (this.metrics != null)
{
this.metrics.OnFileWritten(chunk.Size);
}
chunk.Commit();
}
else
{
if (this.writeCallback == null)
{
this.writeCallback = this.OnWrite;
}
this.fileStream.BeginWrite(chunk.Buffer, chunk.Offset, chunk.Size, this.writeCallback, chunk);
}
}
private void OnWrite(IAsyncResult asyncResult)
{
FileQueue.Chunk chunk = asyncResult.AsyncState as FileQueue.Chunk;
this.fileStream.EndWrite(asyncResult);
if (this.metrics != null)
{
this.metrics.OnFileWritten(chunk.Size);
}
chunk.Commit();
}
}
}

View File

@@ -0,0 +1,270 @@
using System;
using System.Collections.Generic;
using CustomsFramework;
using Server.Guilds;
namespace Server
{
public class StandardSaveStrategy : SaveStrategy
{
public static SaveOption SaveType = SaveOption.Normal;
private readonly Queue<Item> _decayQueue;
private bool _permitBackgroundWrite;
public StandardSaveStrategy()
{
this._decayQueue = new Queue<Item>();
}
public enum SaveOption
{
Normal,
Threaded
}
public override string Name
{
get
{
return "Standard";
}
}
protected bool PermitBackgroundWrite
{
get
{
return this._permitBackgroundWrite;
}
set
{
this._permitBackgroundWrite = value;
}
}
protected bool UseSequentialWriters
{
get
{
return (StandardSaveStrategy.SaveType == SaveOption.Normal || !this._permitBackgroundWrite);
}
}
public override void Save(SaveMetrics metrics, bool permitBackgroundWrite)
{
this._permitBackgroundWrite = permitBackgroundWrite;
this.SaveMobiles(metrics);
this.SaveItems(metrics);
this.SaveGuilds(metrics);
this.SaveData(metrics);
if (permitBackgroundWrite && this.UseSequentialWriters) //If we're permitted to write in the background, but we don't anyways, then notify.
World.NotifyDiskWriteComplete();
}
public override void ProcessDecay()
{
while (this._decayQueue.Count > 0)
{
Item item = this._decayQueue.Dequeue();
if (item.OnDecay())
{
item.Delete();
}
}
}
protected void SaveMobiles(SaveMetrics metrics)
{
Dictionary<Serial, Mobile> mobiles = World.Mobiles;
GenericWriter idx;
GenericWriter tdb;
GenericWriter bin;
if (this.UseSequentialWriters)
{
idx = new BinaryFileWriter(World.MobileIndexPath, false);
tdb = new BinaryFileWriter(World.MobileTypesPath, false);
bin = new BinaryFileWriter(World.MobileDataPath, true);
}
else
{
idx = new AsyncWriter(World.MobileIndexPath, false);
tdb = new AsyncWriter(World.MobileTypesPath, false);
bin = new AsyncWriter(World.MobileDataPath, true);
}
idx.Write((int)mobiles.Count);
foreach (Mobile m in mobiles.Values)
{
long start = bin.Position;
idx.Write((int)m.m_TypeRef);
idx.Write((int)m.Serial);
idx.Write((long)start);
m.Serialize(bin);
if (metrics != null)
{
metrics.OnMobileSaved((int)(bin.Position - start));
}
idx.Write((int)(bin.Position - start));
m.FreeCache();
}
tdb.Write((int)World.m_MobileTypes.Count);
for (int i = 0; i < World.m_MobileTypes.Count; ++i)
tdb.Write(World.m_MobileTypes[i].FullName);
idx.Close();
tdb.Close();
bin.Close();
}
protected void SaveItems(SaveMetrics metrics)
{
Dictionary<Serial, Item> items = World.Items;
GenericWriter idx;
GenericWriter tdb;
GenericWriter bin;
if (this.UseSequentialWriters)
{
idx = new BinaryFileWriter(World.ItemIndexPath, false);
tdb = new BinaryFileWriter(World.ItemTypesPath, false);
bin = new BinaryFileWriter(World.ItemDataPath, true);
}
else
{
idx = new AsyncWriter(World.ItemIndexPath, false);
tdb = new AsyncWriter(World.ItemTypesPath, false);
bin = new AsyncWriter(World.ItemDataPath, true);
}
idx.Write((int)items.Count);
foreach (Item item in items.Values)
{
if (item.Decays && item.Parent == null && item.Map != Map.Internal && (item.LastMoved + item.DecayTime) <= DateTime.UtcNow)
{
this._decayQueue.Enqueue(item);
}
long start = bin.Position;
idx.Write((int)item.m_TypeRef);
idx.Write((int)item.Serial);
idx.Write((long)start);
item.Serialize(bin);
if (metrics != null)
{
metrics.OnItemSaved((int)(bin.Position - start));
}
idx.Write((int)(bin.Position - start));
item.FreeCache();
}
tdb.Write((int)World.m_ItemTypes.Count);
for (int i = 0; i < World.m_ItemTypes.Count; ++i)
tdb.Write(World.m_ItemTypes[i].FullName);
idx.Close();
tdb.Close();
bin.Close();
}
protected void SaveGuilds(SaveMetrics metrics)
{
GenericWriter idx;
GenericWriter bin;
if (this.UseSequentialWriters)
{
idx = new BinaryFileWriter(World.GuildIndexPath, false);
bin = new BinaryFileWriter(World.GuildDataPath, true);
}
else
{
idx = new AsyncWriter(World.GuildIndexPath, false);
bin = new AsyncWriter(World.GuildDataPath, true);
}
idx.Write((int)BaseGuild.List.Count);
foreach (BaseGuild guild in BaseGuild.List.Values)
{
long start = bin.Position;
idx.Write((int)0);//guilds have no typeid
idx.Write((int)guild.Id);
idx.Write((long)start);
guild.Serialize(bin);
if (metrics != null)
{
metrics.OnGuildSaved((int)(bin.Position - start));
}
idx.Write((int)(bin.Position - start));
}
idx.Close();
bin.Close();
}
protected void SaveData(SaveMetrics metrics)
{
Dictionary<CustomSerial, SaveData> data = World.Data;
GenericWriter indexWriter;
GenericWriter typeWriter;
GenericWriter dataWriter;
if (this.UseSequentialWriters)
{
indexWriter = new BinaryFileWriter(World.DataIndexPath, false);
typeWriter = new BinaryFileWriter(World.DataTypesPath, false);
dataWriter = new BinaryFileWriter(World.DataBinaryPath, true);
}
else
{
indexWriter = new AsyncWriter(World.DataIndexPath, false);
typeWriter = new AsyncWriter(World.DataTypesPath, false);
dataWriter = new AsyncWriter(World.DataBinaryPath, true);
}
indexWriter.Write(data.Count);
foreach (SaveData saveData in data.Values)
{
long start = dataWriter.Position;
indexWriter.Write(saveData._TypeID);
indexWriter.Write((int)saveData.Serial);
indexWriter.Write(start);
saveData.Serialize(dataWriter);
if (metrics != null)
metrics.OnDataSaved((int)(dataWriter.Position - start));
indexWriter.Write((int)(dataWriter.Position - start));
}
typeWriter.Write(World._DataTypes.Count);
for (int i = 0; i < World._DataTypes.Count; ++i)
typeWriter.Write(World._DataTypes[i].FullName);
indexWriter.Close();
typeWriter.Close();
dataWriter.Close();
}
}
}