Skip to content

HPCC-35694 Add support to WsDFU.DFUQuery to sort by physical "File Size" #21119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
asselitx wants to merge 1 commit into
base: candidate-10.2.x
Choose a base branch
from
Open

HPCC-35694 Add support to WsDFU.DFUQuery to sort by physical "File Size" #21119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions dali/base/dadfs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14255,11 +14255,16 @@ static IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, unsi

if (options.includeField(DFUQResultField::size))
{
// JCSMORE - I am not sure what the point of this is, with or without it, a blank @size does not affect sort order
// and EclWatch seems to use the @size (DFUQResultField::origsize) for size column anyway
// See special handling in SerializeFileAttrOptions::readFields() to include origsize if size is requested
// Size field is notionally the size on disk
const char *propName = getDFUQResultFieldName(DFUQResultField::size);
attr->setPropInt64(propName, attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1));//Sort the files with empty size to front
attr->setPropInt64(propName, isCompressed(*attr) ? attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::compressedsize), -1) : attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1));
}

if (options.includeField(DFUQResultField::origsize))
{
const char *propName = getDFUQResultFieldName(DFUQResultField::origsize);
if (!attr->hasProp(propName))
attr->setPropInt64(propName, -1);
}

if (options.includeField(DFUQResultField::recordcount))
Expand Down
4 changes: 2 additions & 2 deletions esp/scm/ws_dfu.ecm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1037,8 +1037,8 @@ ESPresponse [version("1.68"), exceptions_inline, nil_remove] DFUFileRenameRespon
// ===========================================================================
ESPservice [
auth_feature("DEFERRED"),
version("1.68"),
default_client_version("1.68"),
version("1.69"),
default_client_version("1.69"),
noforms,
exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
{
Expand Down
2 changes: 1 addition & 1 deletion esp/scm/ws_dfuXref.ecm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ ESPresponse [exceptions_inline] DFUXRefUnusedFilesResponse
};

// ===========================================================================
ESPservice [version("1.04"), default_client_version("1.04"), auth_feature("DEFERRED"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsDFUXRef
ESPservice [version("1.69"), default_client_version("1.69"), auth_feature("DEFERRED"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsDFUXRef
{
///ESPmethod [resp_xsl_default("./smc_xslt/xref_main.xslt")] DFUXRefList(DFUXRefListRequest, DFUXRefListResponse);
ESPmethod [resp_xsl_default("/esp/xslt/xref_main.xslt")] DFUXRefList(DFUXRefListRequest, DFUXRefListResponse);
Expand Down
1 change: 1 addition & 0 deletions esp/scm/ws_dfu_common.ecm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,5 @@ ESPStruct DFULogicalFile
[min_ver("1.63"), nil_remove] int64 MaxSkew;
[min_ver("1.63"), nil_remove] int64 MinSkewPart;
[min_ver("1.63"), nil_remove] int64 MaxSkewPart;
[min_ver("1.69")] int64 FileSize;
};
20 changes: 20 additions & 0 deletions esp/services/ws_dfu/ws_dfuHelpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,26 @@ bool WsDFUHelpers::addToLogicalFileList(IPropertyTree& file, const char* nodeGro
lFile->setMinSkewPart(file.getPropInt64(getDFUQResultFieldName(DFUQResultField::minSkewPart)));
}

if (version >= 1.68)

@ghalliday ghalliday Apr 10, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should be compare >= 1.69

{
if (file.hasProp(getDFUQResultFieldName(DFUQResultField::size)))
lFile->setFileSize(file.getPropInt64(getDFUQResultFieldName(DFUQResultField::size)));
else
{
if (isFileCompressed)
{
if (file.hasProp(getDFUQResultFieldName(DFUQResultField::compressedsize)))
lFile->setFileSize(file.getPropInt64(getDFUQResultFieldName(DFUQResultField::compressedsize)));
else if (isKeyFile)
lFile->setFileSize(size);
}
else
{
lFile->setFileSize(size);
}
}
}

logicalFiles.append(*lFile.getClear());
}
catch(IException* e)
Expand Down
1 change: 1 addition & 0 deletions esp/services/ws_dfu/ws_dfuService.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3769,6 +3769,7 @@ void CWsDfuEx::setDFUQuerySortOrder(IEspDFUQueryRequest& req, StringBuffer& sort
static const std::unordered_map<std::string_view, std::string_view> legacyMappings =
{
{"FileSize", "@DFUSFsize"},
{"CompressedFileSize", "@compressedSize"},
Comment thread
asselitx marked this conversation as resolved.
{"ContentType", "@kind"},
{"IsCompressed", "@compressed"},
{"Records", "@recordcount"},
Expand Down
4 changes: 2 additions & 2 deletions esp/src/eclwatch/DFUQueryWidget.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,7 @@ define([
label: nlsHPCC.MinSkew, width: 60,
formatter: function (value, row) {
if (value) {
return "-" + Utility.formatDecimal(value / 100) + "%";
return Utility.formatDecimal(value / 100, "-", "%");
}
return "";
}
Expand All @@ -746,7 +746,7 @@ define([
label: nlsHPCC.MaxSkew, width: 60,
formatter: function (value, row) {
if (value) {
return Utility.formatDecimal(value / 100) + "%";
return Utility.formatDecimal(value / 100, "", "%");
}
return "";
}
Expand Down
29 changes: 18 additions & 11 deletions esp/src/src-react/components/Files.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { QuerySortItem } from "src/store/Store";
import nlsHPCC from "src/nlsHPCC";
import { useConfirm } from "../hooks/confirm";
import { useMyAccount } from "../hooks/user";
import { formatCompression } from "../hooks/file";
import { HolyGrail } from "../layouts/HolyGrail";
import { pushParams } from "../util/history";
import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid";
Expand Down Expand Up @@ -102,7 +103,6 @@ export const Files: React.FunctionComponent<FilesProps> = ({
page = 1,
store
}) => {

const hasFilter = React.useMemo(() => Object.keys(filter).length > 0, [filter]);

const [showFilter, setShowFilter] = React.useState(false);
Expand Down Expand Up @@ -210,28 +210,35 @@ export const Files: React.FunctionComponent<FilesProps> = ({
csvFormatter: (value, row) => row.IntRecordCount,
},
FileSize: {
label: nlsHPCC.Size,
label: nlsHPCC.FileSize,
formatter: (value, row) => {
return Utility.convertedSize(row.IntSize);
if (row.FileSize !== undefined) {
return Utility.convertedSize(row.FileSize);
}
return Utility.convertedSize(row.IsCompressed ? row.CompressedFileSize : row.IntSize);
},
csvFormatter: (value, row) => {
if (row.FileSize !== undefined) {
return row.FileSize;
}
return row.IsCompressed ? row.CompressedFileSize : row.IntSize;
},
csvFormatter: (value, row) => row.IntSize,
},
CompressedFileSizeString: {
label: nlsHPCC.CompressedSize,
Compression: {
label: nlsHPCC.Compression,
sortable: false,
formatter: (value, row) => {
return Utility.convertedSize(row.CompressedFileSize);
},
csvFormatter: (value, row) => row.CompressedFileSize,
return formatCompression(row);
}
},
Parts: {
label: nlsHPCC.Parts, width: 40,
},
MinSkew: {
label: nlsHPCC.MinSkew, width: 60, formatter: (value, row) => value ? `-${Utility.formatDecimal(value / 100)}%` : ""
label: nlsHPCC.MinSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100, "-", "%")}` : ""
},
MaxSkew: {
label: nlsHPCC.MaxSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100)}%` : ""
label: nlsHPCC.MaxSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100, "", "%")}` : ""
},
Accessed: {
label: uiState.isUTC ? nlsHPCC.LastAccessed : nlsHPCC.LastAccessedLocalTime,
Expand Down
6 changes: 3 additions & 3 deletions esp/src/src-react/components/IndexFileSummary.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,21 +269,21 @@ export const IndexFileSummary: React.FunctionComponent<IndexFileSummaryProps> =
label: nlsHPCC.File,
originalSize: Utility.convertedSize(file?.FileSizeInt64),
diskSize: Utility.convertedSize(file?.CompressedFileSize || file?.FileSizeInt64),
percentCompressed: ((file?.CompressedFileSize && file?.FileSizeInt64) ? Utility.formatDecimal(100 * file?.CompressedFileSize / file?.FileSizeInt64) : 0) + "%",
percentCompressed: ((file?.CompressedFileSize && file?.FileSizeInt64) ? Utility.formatDecimal(100 * file?.CompressedFileSize / file?.FileSizeInt64, "", "%") : ""),
memorySize: (file?.ExtendedIndexInfo?.SizeMemoryBranches && file?.ExtendedIndexInfo?.SizeMemoryLeaves) ? Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryBranches + file?.ExtendedIndexInfo?.SizeMemoryLeaves) : ""
},
{
label: nlsHPCC.Branches,
originalSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeOriginalBranches) ?? "",
diskSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeDiskBranches) ?? "",
percentCompressed: file?.ExtendedIndexInfo?.BranchCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.BranchCompressionPercent) + "%" : "",
percentCompressed: file?.ExtendedIndexInfo?.BranchCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.BranchCompressionPercent, "", "%") : "",
memorySize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryBranches) ?? ""
},
{
label: nlsHPCC.Data,
originalSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeOriginalData) ?? "",
diskSize: (file?.ExtendedIndexInfo?.SizeDiskLeaves !== undefined && file?.ExtendedIndexInfo?.SizeDiskBlobs !== undefined) ? Utility.convertedSize(file?.ExtendedIndexInfo?.SizeDiskLeaves + file?.ExtendedIndexInfo?.SizeDiskBlobs) : "",
percentCompressed: file?.ExtendedIndexInfo?.DataCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.DataCompressionPercent) + "%" : "",
percentCompressed: file?.ExtendedIndexInfo?.DataCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.DataCompressionPercent, "", "%") : "",
memorySize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryLeaves) ?? ""
}
]}
Expand Down
6 changes: 3 additions & 3 deletions esp/src/src-react/components/LogicalFileSummary.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { formatCost } from "src/Session";
import * as Utility from "src/Utility";
import { getStateImageName, IFile } from "src/ESPLogicalFile";
import { useConfirm } from "../hooks/confirm";
import { useFile } from "../hooks/file";
import { formatCompression, useFile } from "../hooks/file";
import { useMyAccount } from "../hooks/user";
import { ShortVerticalDivider } from "./Common";
import { TableGroup } from "./forms/Groups";
Expand Down Expand Up @@ -218,11 +218,11 @@ export const LogicalFileSummary: React.FunctionComponent<LogicalFileSummaryProps
"ContentType": { label: nlsHPCC.ContentType, type: "string", value: file?.ContentType, readonly: true },
"KeyType": { label: nlsHPCC.KeyType, type: "string", value: file?.KeyType, readonly: true },
"Format": { label: nlsHPCC.Format, type: "string", value: file?.Format, readonly: true },
"Filesize": { label: nlsHPCC.FileSize, type: "string", value: file?.Filesize, readonly: true },
"IsCompressed": { label: nlsHPCC.IsCompressed, type: "checkbox", value: file?.IsCompressed, readonly: true },
"CompressedFileSizeString": { label: nlsHPCC.CompressedFileSize, type: "string", value: file?.CompressedFileSize ? Utility.safeFormatNum(file?.CompressedFileSize) : "", readonly: true },
"CompressionType": { label: nlsHPCC.CompressionType, type: "string", value: file?.CompressionType, readonly: true },
"Filesize": { label: nlsHPCC.FileSize, type: "string", value: file?.Filesize, readonly: true },
"PercentCompressed": { label: nlsHPCC.PercentCompressed, type: "string", value: file?.PercentCompressed, readonly: true },
"Compression": { label: nlsHPCC.Compression, type: "string", value: formatCompression(file), readonly: true },
"Modified": { label: nlsHPCC.Modified, type: "string", value: file?.Modified, readonly: true },
"ExpirationDate": { label: nlsHPCC.ExpirationDate, type: "string", value: file?.ExpirationDate, readonly: true },
"ExpireDays": { label: nlsHPCC.ExpireDays, type: "string", value: file?.ExpireDays ? file?.ExpireDays.toString() : "", readonly: true },
Expand Down
2 changes: 1 addition & 1 deletion esp/src/src-react/components/MetricsPropertiesTables.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT
if (row.SkewMax !== undefined) parts.push(`⤒${row.SkewMax}`);
rowValue = parts.join(" ");
}
scopeProps.push([row.Key, rowValue, row.Avg, row.Min, row.Max, row.Delta, row.StdDev === undefined ? "" : `${row.StdDev} (${formatDecimal(row.StdDevs)}σ)`, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax, row.StdDevs]);
scopeProps.push([row.Key, rowValue, row.Avg, row.Min, row.Max, row.Delta, row.StdDev === undefined ? "" : `${row.StdDev} ${formatDecimal(row.StdDevs, "(", "σ)")}`, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax, row.StdDevs]);
}
scopeProps.sort((l, r) => {
const lIdx = sortByColumns.indexOf(l[0]);
Expand Down
28 changes: 28 additions & 0 deletions esp/src/src-react/hooks/file.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,39 @@
import * as React from "react";
import { LogicalFile, WsDfu } from "@hpcc-js/comms";
import { scopedLogger } from "@hpcc-js/util";
import * as Utility from "src/Utility";
import { singletonDebounce } from "../util/throttle";
import { useCounter } from "./util";

const logger = scopedLogger("../hooks/file.ts");

export interface LogicalFileEx extends LogicalFile {
IntSize: number;
}

function isLogicalFileEx(file: LogicalFile): file is LogicalFileEx {
return (file as LogicalFileEx).IntSize !== undefined;
}

function formatRatio(isCompressed: boolean, compressedSize: number, totalSize: number): string {
if (!isCompressed || totalSize <= 0) return "";
const ratio = compressedSize / totalSize;
if (!isFinite(ratio)) return "";
return Utility.formatDecimal(100 - ratio * 100, "", "%");
}
Comment on lines +18 to +23

function formatCompressionEx(file?: LogicalFileEx): string {
if (!file) return "";
return formatRatio(file.IsCompressed, file.CompressedFileSize, file.IntSize);
}

export function formatCompression(file?: LogicalFile): string {
if (!file) return "";
if (file.isSuperfile) return "";
if (isLogicalFileEx(file)) return formatCompressionEx(file);
return formatRatio(file.IsCompressed, file.CompressedFileSize, file.FileSizeInt64);
}

interface useFileResponse {
file: LogicalFile;
protectedBy: WsDfu.DFUFileProtect[];
Expand Down
4 changes: 2 additions & 2 deletions esp/src/src/Utility.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1191,10 +1191,10 @@ export function deleteCookie(name: string) {
const d3FormatDecimal = d3Format(",.2f");
const d3FormatInt = d3Format(",.0f");

export function formatDecimal(num: number): string {
export function formatDecimal(num: number, prefix: string = "", postfix: string = ""): string {
if (!num) return "";
if (isNaN(num)) return num.toString();
return d3FormatDecimal(num);
return prefix + d3FormatDecimal(num) + postfix;
Comment on lines +1194 to +1197
}

export function formatNum(num: number): string {
Expand Down
4 changes: 2 additions & 2 deletions esp/src/tests/v9-files.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ test.describe("V9 Files - Logical Files", () => {
await expect(page.getByText("Description")).toBeVisible();
await expect(page.getByText("Cluster", { exact: true })).toBeVisible();
await expect(page.getByText("Records")).toBeVisible();
await expect(page.getByText("Size", { exact: true })).toBeVisible();
await expect(page.getByText("Compressed Size")).toBeVisible();
await expect(page.getByText("File Size")).toBeVisible();
await expect(page.getByText("Compression")).toBeVisible();
await expect(page.getByText("Parts")).toBeVisible();
await expect(page.getByText("Min Skew")).toBeVisible();
await expect(page.getByText("Max Skew")).toBeVisible();
Expand Down
13 changes: 12 additions & 1 deletion testing/unittests/dalitests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2683,6 +2683,9 @@ class DaliDFSIteratorTests : public CppUnit::TestFixture
attr.setProp("@job", attrValue);
attr.setProp("@owner", attrValue);
attr.setProp("@workunit", attrValue);
attr.setPropBool("@rowCompressed", true);
attr.setPropInt64("@compressedSize", 9);
attr.setPropInt64("@size", 17);
}
}

Expand Down Expand Up @@ -2835,7 +2838,12 @@ class DaliDFSIteratorTests : public CppUnit::TestFixture
filterBuf.append(DFUQFTspecial).append(DFUQFilterSeparator).append(DFUQSFFileType).append(DFUQFilterSeparator).append(DFUQFFTnonsuperfileonly).append(DFUQFilterSeparator);
filterBuf.append(DFUQFTspecial).append(DFUQFilterSeparator).append(DFUQSFFileNameWithPrefix).append(DFUQFilterSeparator).append(wildName).append(DFUQFilterSeparator);

std::vector<DFUQResultField> fields = {DFUQResultField::size, DFUQResultField::cost, DFUQResultField::term};
// Must include all fields needed for sorting and inspection in loop below.
//
// rowCompressed and blockCompressed required for getLogicalFilesSorted to successfully run the isCompressed()
// test used when determining which size value to populate DFUQResultField::size with (compressed vs uncompressed).
// Alternately, could use DFUQResultField::includeAll to return all fields if the test expands in scope.
std::vector<DFUQResultField> fields = {DFUQResultField::size, DFUQResultField::cost, DFUQResultField::rowCompressed, DFUQResultField::blockCompressed, DFUQResultField::term};

DFUQResultField sortOrder[2] = {DFUQResultField::job|DFUQResultField::reverse, DFUQResultField::term};
__int64 hint;
Expand All @@ -2857,6 +2865,9 @@ class DaliDFSIteratorTests : public CppUnit::TestFixture
CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: Missing cost attributes", costAttrsPresent);
bool dirAttrsPresent = attrs.hasProp("@directory");
CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: directory attribute should NOT be present", !dirAttrsPresent);
CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: size field missing", attrs.hasProp("@DFUSFsize"));
CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: compressed size field missing", attrs.hasProp("@compressedSize"));
CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: size should use compressed size", attrs.getPropInt64("@DFUSFsize", -1) == attrs.getPropInt64("@compressedSize", -1));
}
}
};
Expand Down
Loading