TradeSkillMaster/Core/Service/AuctionDB/Core.lua

572 lines
20 KiB
Lua

-- ------------------------------------------------------------------------------ --
-- TradeSkillMaster --
-- https://tradeskillmaster.com --
-- All Rights Reserved - Detailed license information included with addon. --
-- ------------------------------------------------------------------------------ --
local _, TSM = ...
local AuctionDB = TSM:NewPackage("AuctionDB")
local L = TSM.Include("Locale").GetTable()
local Event = TSM.Include("Util.Event")
local CSV = TSM.Include("Util.CSV")
local Table = TSM.Include("Util.Table")
local Math = TSM.Include("Util.Math")
local Log = TSM.Include("Util.Log")
local ItemString = TSM.Include("Util.ItemString")
local Wow = TSM.Include("Util.Wow")
local Threading = TSM.Include("Service.Threading")
local ItemInfo = TSM.Include("Service.ItemInfo")
local CustomPrice = TSM.Include("Service.CustomPrice")
local AuctionScan = TSM.Include("Service.AuctionScan")
local private = {
region = nil,
realmAppData = {
scanTime = nil,
data = {},
itemOffset = {},
fieldOffset = {},
numFields = nil,
},
regionData = nil,
regionUpdateTime = nil,
scanRealmData = {},
scanRealmTime = nil,
scanThreadId = nil,
ahOpen = false,
didScan = false,
auctionScan = nil,
isScanning = false,
}
local CSV_KEYS = { "itemString", "minBuyout", "marketValue", "numAuctions", "quantity", "lastScan" }
-- ============================================================================
-- Module Functions
-- ============================================================================
function AuctionDB.OnInitialize()
private.scanThreadId = Threading.New("AUCTIONDB_SCAN", private.ScanThread)
Threading.SetCallback(private.scanThreadId, private.ScanThreadCleanup)
Event.Register("AUCTION_HOUSE_SHOW", private.OnAuctionHouseShow)
Event.Register("AUCTION_HOUSE_CLOSED", private.OnAuctionHouseClosed)
end
function AuctionDB.OnEnable()
private.region = TSM.GetRegion()
local realmAppData = nil
local appData = TSMAPI.AppHelper and TSMAPI.AppHelper:FetchData("AUCTIONDB_MARKET_DATA") -- get app data from TSM_AppHelper if it's installed
if appData then
for _, info in ipairs(appData) do
local realm, data = unpack(info)
local downloadTime = "?"
-- try switching around "Classic-[US|EU]" to match the addon's "[US|EU]-Classic" format for classic region data
if realm == private.region or gsub(realm, "Classic-%-([A-Z]+)", "%1-Classic") == private.region then
private.regionData, private.regionUpdateTime = private.LoadRegionAppData(data)
downloadTime = SecondsToTime(time() - private.regionUpdateTime).." ago"
elseif TSMAPI.AppHelper:IsCurrentRealm(realm) then
realmAppData = private.ProcessRealmAppData(data)
downloadTime = SecondsToTime(time() - realmAppData.downloadTime).." ago"
end
Log.Info("Got AppData for %s (isCurrent=%s, %s)", realm, tostring(TSMAPI.AppHelper:IsCurrentRealm(realm)), downloadTime)
end
end
-- check if we can load realm data from the app
if realmAppData then
private.realmAppData.scanTime = realmAppData.downloadTime
for i = 2, #realmAppData.fields do
private.realmAppData.fieldOffset[realmAppData.fields[i]] = i - 1
end
private.realmAppData.numFields = #realmAppData.fields - 1
local numRawFields = #realmAppData.fields
local nextItmeOffset, nextDataOffset = 0, 1
for _, data in ipairs(realmAppData.data) do
for i = 1, numRawFields do
local value = data[i]
if i == 1 then
-- item string must be the first field
local itemString = nil
if type(value) == "number" then
itemString = "i:"..value
else
itemString = gsub(value, ":0:", "::")
end
itemString = ItemString.Get(itemString)
private.realmAppData.itemOffset[itemString] = nextItmeOffset
nextItmeOffset = nextItmeOffset + 1
else
private.realmAppData.data[nextDataOffset] = value
nextDataOffset = nextDataOffset + 1
end
end
end
end
for itemString in pairs(private.realmAppData.itemOffset) do
ItemInfo.FetchInfo(itemString)
end
if TSM.db.factionrealm.internalData.auctionDBScanTime > 0 then
private.LoadSVRealmData()
end
if not private.realmAppData.numFields and not next(private.scanRealmData) then
Log.PrintfUser(L["TSM doesn't currently have any AuctionDB pricing data for your realm. We recommend you download the TSM Desktop Application from %s to automatically update your AuctionDB data (and auto-backup your TSM settings)."], Log.ColorUserAccentText("https://tradeskillmaster.com"))
end
CustomPrice.OnSourceChange("DBMarket")
CustomPrice.OnSourceChange("DBMinBuyout")
CustomPrice.OnSourceChange("DBHistorical")
CustomPrice.OnSourceChange("DBRegionMinBuyoutAvg")
CustomPrice.OnSourceChange("DBRegionMarketAvg")
CustomPrice.OnSourceChange("DBRegionHistorical")
CustomPrice.OnSourceChange("DBRegionSaleAvg")
CustomPrice.OnSourceChange("DBRegionSaleRate")
CustomPrice.OnSourceChange("DBRegionSoldPerDay")
collectgarbage()
end
function AuctionDB.OnDisable()
if not private.didScan then
return
end
local encodeContext = CSV.EncodeStart(CSV_KEYS)
for itemString, data in pairs(private.scanRealmData) do
CSV.EncodeAddRowDataRaw(encodeContext, itemString, data.minBuyout, data.marketValue, data.numAuctions, data.quantity, data.lastScan)
end
TSM.db.factionrealm.internalData.csvAuctionDBScan = CSV.EncodeEnd(encodeContext)
TSM.db.factionrealm.internalData.auctionDBScanHash = Math.CalculateHash(TSM.db.factionrealm.internalData.csvAuctionDBScan)
end
function AuctionDB.GetAppDataUpdateTimes()
return private.realmAppData.scanTime or 0, private.regionUpdateTime or 0
end
function AuctionDB.GetLastCompleteScanTime()
local result = private.didScan and (private.scanRealmTime or 0) or (private.realmAppData.scanTime or 0)
return result ~= 0 and result or nil
end
function AuctionDB.LastScanIteratorThreaded()
local itemNumAuctions = Threading.AcquireSafeTempTable()
local itemMinBuyout = Threading.AcquireSafeTempTable()
local baseItems = Threading.AcquireSafeTempTable()
local lastScanTime = AuctionDB.GetLastCompleteScanTime()
for itemString, data in pairs(private.didScan and private.scanRealmData or private.realmAppData.itemOffset) do
if not private.didScan or data.lastScan >= lastScanTime then
itemString = ItemString.Get(itemString)
local baseItemString = ItemString.GetBaseFast(itemString)
if baseItemString ~= itemString then
baseItems[baseItemString] = true
end
local numAuctions, minBuyout = nil, nil
if private.didScan then
numAuctions = data.numAuctions
minBuyout = data.minBuyout
else
numAuctions = private.realmAppData.data[data * private.realmAppData.numFields + private.realmAppData.fieldOffset.numAuctions]
minBuyout = private.realmAppData.data[data * private.realmAppData.numFields + private.realmAppData.fieldOffset.minBuyout]
end
itemNumAuctions[itemString] = (itemNumAuctions[itemString] or 0) + numAuctions
if minBuyout and minBuyout > 0 then
itemMinBuyout[itemString] = min(itemMinBuyout[itemString] or math.huge, minBuyout)
end
end
Threading.Yield()
end
-- remove the base items since they would be double-counted with the specific variants
for itemString in pairs(baseItems) do
itemNumAuctions[itemString] = nil
itemMinBuyout[itemString] = nil
end
Threading.ReleaseSafeTempTable(baseItems)
-- convert the remaining items into a list
local itemList = Threading.AcquireSafeTempTable()
itemList.numAuctions = itemNumAuctions
itemList.minBuyout = itemMinBuyout
for itemString in pairs(itemNumAuctions) do
tinsert(itemList, itemString)
end
return Table.Iterator(itemList, private.LastScanIteratorHelper, itemList, private.LastScanIteratorCleanup)
end
function AuctionDB.GetRealmItemData(itemString, key)
local realmData = nil
if private.didScan and (key == "minBuyout" or key == "numAuctions" or key == "lastScan") then
-- always use scanRealmData for minBuyout/numAuctions/lastScan if we've done a scan
realmData = private.scanRealmData
elseif private.realmAppData.numFields then
-- use app data
return private.GetRealmAppItemDataHelper(private.realmAppData, key, itemString)
else
realmData = private.scanRealmData
end
return private.GetItemDataHelper(realmData, key, itemString)
end
function AuctionDB.GetRegionItemData(itemString, key)
return private.GetRegionItemDataHelper(private.regionData, key, itemString)
end
function AuctionDB.GetRegionSaleInfo(itemString, key)
-- need to divide the result by 100
local result = private.GetRegionItemDataHelper(private.regionData, key, itemString)
return result and (result / 100) or nil
end
function AuctionDB.RunScan()
if private.isScanning then
return
end
if not private.ahOpen then
Log.PrintUser(L["ERROR: The auction house must be open in order to do a scan."])
return
end
local canScan, canGetAllScan = CanSendAuctionQuery()
if not canScan then
Log.PrintUser(L["ERROR: The AH is currently busy with another scan. Please try again once that scan has completed."])
return
elseif not canGetAllScan then
Log.PrintUser(L["ERROR: A full AH scan has recently been performed and is on cooldown. Log out to reset this cooldown."])
return
end
if not TSM.UI.AuctionUI.StartingScan("FULL_SCAN") then
return
end
Log.PrintUser(L["Starting full AH scan. Please note that this scan may cause your game client to lag or crash. This scan generally takes 1-2 minutes."])
Threading.Start(private.scanThreadId)
private.isScanning = true
end
-- ============================================================================
-- Scan Thread
-- ============================================================================
function private.ScanThread()
assert(not private.auctionScan)
-- run the scan
local auctionScan = AuctionScan.GetManager()
:SetResolveSellers(false)
private.auctionScan = auctionScan
local query = auctionScan:NewQuery()
:SetGetAll(true)
if not auctionScan:ScanQueriesThreaded() then
Log.PrintUser(L["Failed to run full AH scan."])
return
end
-- process the results
Log.PrintfUser(L["Processing scan results..."])
wipe(private.scanRealmData)
private.scanRealmTime = time()
TSM.db.factionrealm.internalData.auctionDBScanTime = time()
TSM.db.factionrealm.internalData.csvAuctionDBScan = ""
local numScannedAuctions = 0
local subRows = Threading.AcquireSafeTempTable()
local subRowSortValue = Threading.AcquireSafeTempTable()
local itemBuyouts = Threading.AcquireSafeTempTable()
for baseItemString, row in query:BrowseResultsIterator() do
wipe(subRows)
wipe(subRowSortValue)
for _, subRow in row:SubRowIterator() do
local _, itemBuyout = subRow:GetBuyouts()
tinsert(subRows, subRow)
subRowSortValue[subRow] = itemBuyout
end
Table.SortWithValueLookup(subRows, subRowSortValue, false, true)
wipe(itemBuyouts)
for _, subRow in ipairs(subRows) do
local _, itemBuyout = subRow:GetBuyouts()
local quantity, numAuctions = subRow:GetQuantities()
numScannedAuctions = numScannedAuctions + numAuctions
for _ = 1, numAuctions do
private.ProcessScanResultItem(baseItemString, itemBuyout, quantity)
end
if itemBuyout > 0 then
for _ = 1, quantity * numAuctions do
tinsert(itemBuyouts, itemBuyout)
end
end
end
local data = private.scanRealmData[baseItemString]
data.marketValue = private.CalculateItemMarketValue(itemBuyouts, data.quantity)
assert(data.minBuyout == 0 or data.marketValue >= data.minBuyout)
Threading.Yield()
end
Threading.ReleaseSafeTempTable(subRows)
Threading.ReleaseSafeTempTable(subRowSortValue)
Threading.ReleaseSafeTempTable(itemBuyouts)
Threading.Yield()
collectgarbage()
Log.PrintfUser(L["Completed full AH scan (%d auctions)!"], numScannedAuctions)
private.didScan = true
CustomPrice.OnSourceChange("DBMinBuyout")
end
function private.ScanThreadCleanup()
private.isScanning = false
if private.auctionScan then
private.auctionScan:Release()
private.auctionScan = nil
end
TSM.UI.AuctionUI.EndedScan("FULL_SCAN")
end
function private.ProcessScanResultItem(itemString, itemBuyout, stackSize)
private.scanRealmData[itemString] = private.scanRealmData[itemString] or { numAuctions = 0, quantity = 0, minBuyout = 0 }
local data = private.scanRealmData[itemString]
data.lastScan = time()
if itemBuyout > 0 then
data.minBuyout = min(data.minBuyout > 0 and data.minBuyout or math.huge, itemBuyout)
data.quantity = data.quantity + stackSize
end
data.numAuctions = data.numAuctions + 1
end
function private.CalculateItemMarketValue(itemBuyouts, quantity)
assert(#itemBuyouts == quantity)
if quantity == 0 then
return 0
end
-- calculate the average of the lowest 15-30% of auctions
local total, num = 0, 0
local lowBucketNum = max(floor(quantity * 0.15), 1)
local midBucketNum = max(floor(quantity * 0.30), 1)
local prevItemBuyout = 0
for i = 1, midBucketNum do
local itemBuyout = itemBuyouts[i]
if num < lowBucketNum or itemBuyout < prevItemBuyout * 1.2 then
num = num + 1
total = total + itemBuyout
end
prevItemBuyout = itemBuyout
end
local avg = total / num
-- calculate the stdev of the auctions we used in the average
local stdev = nil
if num > 1 then
local stdevSum = 0
for i = 1, num do
local itemBuyout = itemBuyouts[i]
stdevSum = stdevSum + (itemBuyout - avg) ^ 2
end
stdev = sqrt(stdevSum / (num - 1))
else
stdev = 0
end
-- calculate the market value as the average of all data within 1.5 stdev of our previous average
local minItemBuyout = avg - stdev * 1.5
local maxItemBuyout = avg + stdev * 1.5
local avgTotal, avgCount = 0, 0
for i = 1, num do
local itemBuyout = itemBuyouts[i]
if itemBuyout >= minItemBuyout and itemBuyout <= maxItemBuyout then
avgTotal = avgTotal + itemBuyout
avgCount = avgCount + 1
end
end
return avgTotal > 0 and floor(avgTotal / avgCount) or 0
end
-- ============================================================================
-- Private Helper Functions
-- ============================================================================
function private.LoadSVRealmData()
local decodeContext = CSV.DecodeStart(TSM.db.factionrealm.internalData.csvAuctionDBScan, CSV_KEYS)
if not decodeContext then
Log.Err("Failed to decode records")
return
end
for itemString, minBuyout, marketValue, numAuctions, quantity, lastScan in CSV.DecodeIterator(decodeContext) do
private.scanRealmData[itemString] = {
minBuyout = tonumber(minBuyout),
marketValue = tonumber(marketValue),
numAuctions = tonumber(numAuctions),
quantity = tonumber(quantity),
lastScan = tonumber(lastScan),
}
end
if not CSV.DecodeEnd(decodeContext) then
Log.Err("Failed to decode records")
end
private.scanRealmTime = TSM.db.factionrealm.internalData.auctionDBScanTime
end
function private.ProcessRealmAppData(rawData)
if #rawData < 3500000 then
-- we can safely just use loadstring() for strings below 3.5M
return assert(loadstring(rawData)())
end
-- load the data in chunks
local leader, itemData, trailer = strmatch(rawData, "^(.+)data={({.+})}(.+)$")
local resultData = {}
local chunkStart, chunkEnd, nextChunkStart = 1, nil, nil
while chunkStart do
chunkEnd, nextChunkStart = strfind(itemData, "},{", chunkStart + 3400000)
local chunkData = assert(loadstring("return {"..strsub(itemData, chunkStart, chunkEnd).."}")())
for _, data in ipairs(chunkData) do
tinsert(resultData, data)
end
chunkStart = nextChunkStart
end
__AUCTIONDB_IMPORT_TEMP = resultData
local result = assert(loadstring(leader.."data=__AUCTIONDB_IMPORT_TEMP"..trailer)())
__AUCTIONDB_IMPORT_TEMP = nil
return result
end
function private.LoadRegionAppData(appData)
local metaDataEndIndex, dataStartIndex = strfind(appData, ",data={")
local itemData = strsub(appData, dataStartIndex + 1, -3)
local metaDataStr = strsub(appData, 1, metaDataEndIndex - 1).."}"
local metaData = assert(loadstring(metaDataStr))()
local result = { fieldLookup = {}, itemLookup = {} }
for i, field in ipairs(metaData.fields) do
result.fieldLookup[field] = i
end
for itemString, otherData in gmatch(itemData, "{([^,]+),([^}]+)}") do
if tonumber(itemString) then
itemString = "i:"..itemString
else
itemString = gsub(strsub(itemString, 2, -2), ":0:", "::")
end
result.itemLookup[itemString] = otherData
end
return result, metaData.downloadTime
end
function private.LastScanIteratorHelper(index, itemString, tbl)
return index, itemString, tbl.numAuctions[itemString], tbl.minBuyout[itemString]
end
function private.LastScanIteratorCleanup(tbl)
Threading.ReleaseSafeTempTable(tbl.numAuctions)
Threading.ReleaseSafeTempTable(tbl.minBuyout)
Threading.ReleaseSafeTempTable(tbl)
end
function private.GetItemDataHelper(tbl, key, itemString)
if not itemString or not tbl then
return nil
end
itemString = ItemString.Filter(itemString)
local value = nil
if not tbl[itemString] and not strmatch(itemString, "^[ip]:[0-9]+$") then
-- for items with random enchants or for pets, get data for the base item
itemString = private.GetBaseItemHelper(itemString)
end
if not itemString or not tbl[itemString] then
return nil
end
value = tbl[itemString][key]
return (value or 0) > 0 and value or nil
end
function private.GetRegionItemDataHelper(tbl, key, itemString)
if not itemString or not tbl then
return nil
end
itemString = ItemString.Filter(itemString)
local fieldIndex = tbl.fieldLookup[key] - 1
assert(fieldIndex and fieldIndex > 0)
local data = tbl.itemLookup[itemString]
if not data and not strmatch(itemString, "^[ip]:[0-9]+$") then
-- for items with random enchants or for pets, get data for the base item
itemString = private.GetBaseItemHelper(itemString)
itemString = ItemString.GetBase(itemString)
if not itemString then
return nil
end
data = tbl.itemLookup[itemString]
end
if type(data) == "string" then
local tblData = {strsplit(",", data)}
for i = 1, #tblData do
tblData[i] = tonumber(tblData[i])
end
tbl.itemLookup[itemString] = tblData
data = tblData
end
if not data then
return nil
end
local value = data[fieldIndex]
return (value or 0) > 0 and value or nil
end
function private.GetRealmAppItemDataHelper(appData, key, itemString)
if not itemString or not appData.numFields then
return nil
elseif key == "lastScan" then
return appData.scanTime
end
itemString = ItemString.Filter(itemString)
if not appData.itemOffset[itemString] and not strmatch(itemString, "^[ip]:[0-9]+$") then
-- for items with random enchants or for pets, get data for the base item
itemString = private.GetBaseItemHelper(itemString)
if not itemString then
return nil
end
end
if not appData.itemOffset[itemString] then
return nil
end
local value = appData.data[appData.itemOffset[itemString] * appData.numFields + appData.fieldOffset[key]]
return (value or 0) > 0 and value or nil
end
function private.GetBaseItemHelper(itemString)
local quality = ItemInfo.GetQuality(itemString)
local itemLevel = ItemInfo.GetItemLevel(itemString)
local classId = ItemInfo.GetClassId(itemString)
if quality and quality >= 2 and itemLevel and itemLevel >= TSM.CONST.MIN_BONUS_ID_ITEM_LEVEL and (classId == LE_ITEM_CLASS_WEAPON or classId == LE_ITEM_CLASS_ARMOR) then
if strmatch(itemString, "^i:[0-9]+:[0-9%-]*:") then
return nil
end
end
return ItemString.GetBaseFast(itemString)
end
function private.OnAuctionHouseShow()
private.ahOpen = true
if not TSM.IsWowClassic() or not select(2, CanSendAuctionQuery()) then
return
elseif (AuctionDB.GetLastCompleteScanTime() or 0) > time() - 60 * 60 * 2 then
-- the most recent scan is from the past 2 hours
return
elseif (TSM.db.factionrealm.internalData.auctionDBScanTime or 0) > time() - 60 * 60 * 24 then
-- this user has contributed a scan within the past 24 hours
return
end
StaticPopupDialogs["TSM_AUCTIONDB_SCAN"] = StaticPopupDialogs["TSM_AUCTIONDB_SCAN"] or {
text = L["TSM does not have recent AuctionDB data. Would you like to run a full AH scan?"],
button1 = YES,
button2 = NO,
timeout = 0,
OnAccept = AuctionDB.RunScan,
}
Wow.ShowStaticPopupDialog("TSM_AUCTIONDB_SCAN")
end
function private.OnAuctionHouseClosed()
private.ahOpen = false
end