feat(ENGKNOW-2274): versioned link files phase 2

.gitignore

@@ -20,6 +20,8 @@
 **/.run/
 
 **/.vscode/
+**/.kiro/
+**/.zed/
 
 **/.project
 **/.classpath

documentation/src/all_commands.rst

@@ -118,4 +118,5 @@
    command/LOG
    command/TRYSELECT
    command/TRYWHERE
-   command/TRYHIDE
+   command/TRYHIDE
+   command/TSVAPPEND

documentation/src/command/TSVAPPEND.rst

@@ -0,0 +1,43 @@
+.. raw:: html
+
+   <span style="float:right; padding-top:10px; color:#BABABA;">Used in: nor</span>
+
+.. _TSVAPPEND:
+
+=====
+TSVAPPEND
+=====
+The :ref:`TSVAPPEND` command can be used to append a stream to a file.
+
+Usage
+=====
+
+.. code-block:: gor
+
+	nor ... | tsvappend <file name> [-noheader] [-prefix <prefix>] [-link <link path>]
+
+Options
+=======
+
++-------------------+-----------------------------------------------------------------+
+| ``-prefix <hf>``  | Takes in a text source containing prefix to be prepended to the |
+|                   | file written. Also support string in single quotes              |
++-------------------+-----------------------------------------------------------------+
+| ``-noheader``     | Don't write a header lines.  Not valid with gor/gorz/nor/norz.  |
++-------------------+-----------------------------------------------------------------+
+| ``-link <link>``  | Writes a link file pointing to the the <file name>.             |
+|                   |                                                                 |
++-------------------+-----------------------------------------------------------------+
+| ``-vlink <link>`` | Writes a versioned link file pointing to the the <file name>.   |
+|                   | The <file name> should not be overwritten if it has previously  |
+|                   | been used in a link file.                                       |
++-------------------+-----------------------------------------------------------------+
+
+Examples
+========
+
+.. code-block:: gor
+
+    nor fileA.nor | tsvappend fileB.nor
+
+The query above will apppend the contents of ``fileA.gor`` to the file ``fileB.gor``.

documentation/src/command/WRITE.rst

@@ -22,46 +22,53 @@ Usage
 
 .. code-block:: gor
 
-	gor ... | write filename [-f forkCol] [-d] [-r] [-c] [-m] [-i type]
+	gor ... | write <file name> [-f forkCol] [-d] [-r] [-c] [-m] [-i type]
 
 Options
 =======
 
-+-----------------+-----------------------------------------------------------------+
-| ``-f column``   | The "fork column" used to split the output into multiple files. |
-+-----------------+-----------------------------------------------------------------+
-| ``-d``          | Use subdirectories instead of #{fork} in filename for forkwrite.|
-+-----------------+-----------------------------------------------------------------+
-| ``-r``          | Eliminate the fork column from the output.                      |
-+-----------------+-----------------------------------------------------------------+
-| ``-c``          | Use column store compression for the output.                    |
-+-----------------+-----------------------------------------------------------------+
-| ``-m``          | Create MD5 sum file along with the output file.                 |
-+-----------------+-----------------------------------------------------------------+
-| ``-maxseg``     | Write maxseg to the gor meta file.                              |
-+-----------------+-----------------------------------------------------------------+
-| ``-inferschema``| Write schema to the gor meta file.                              |
-+-----------------+-----------------------------------------------------------------+
-| ``-i type``     | Write index file (.gori) with a .gorz file, (.tbi) with .vcf.gz |
-|                 | Must state the type, which can be FULL, CHROM or TABIX          |
-+-----------------+-----------------------------------------------------------------+
-| ``-l level``    | Compression level (0-9). Default 1.                             |
-+-----------------+-----------------------------------------------------------------+
-| ``-t 'tags'``   | List of tags which write ensures a file will be created.        |
-|                 | Only valid with the -f option.                                  |
-+-----------------+-----------------------------------------------------------------+
-| ``-tags 'tags'``| List of tags/alias to use in the resulting dictionary when      |
-|                 | writing the files to directories.   Usually used with partgor   |
-|                 | as ``-tags #{tags}``.                                           |
-+-----------------+-----------------------------------------------------------------+
-| ``-prefix hf``  | Takes in a text source containing prefix to be prepended to the |
-|                 | file written. Also support string in single quotes              |
-+-----------------+-----------------------------------------------------------------+
-| ``-noheader``   | Don't write a header lines.  Not valid with gor/gorz/nor/norz.  |
-+-----------------+-----------------------------------------------------------------+
-| ``-card 'cols'``| Calculate cardinality of columns in 'cols' and adds to the      |
-|                 | outputs meta data.                                              |
-+-----------------+-----------------------------------------------------------------+
++-------------------+-----------------------------------------------------------------+
+| ``-f <column>``   | The "fork column" used to split the output into multiple files. |
++-------------------+-----------------------------------------------------------------+
+| ``-d``            | Use subdirectories instead of #{fork} in filename for forkwrite.|
++-------------------+-----------------------------------------------------------------+
+| ``-r``            | Eliminate the fork column from the output.                      |
++-------------------+-----------------------------------------------------------------+
+| ``-c``            | Use column store compression for the output.                    |
++-------------------+-----------------------------------------------------------------+
+| ``-m``            | Create MD5 sum file along with the output file.                 |
++-------------------+-----------------------------------------------------------------+
+| ``-maxseg``       | Write maxseg to the gor meta file.                              |
++-------------------+-----------------------------------------------------------------+
+| ``-inferschema``  | Write schema to the gor meta file.                              |
++-------------------+-----------------------------------------------------------------+
+| ``-i <type>``     | Write index file (.gori) with a .gorz file, (.tbi) with .vcf.gz |
+|                   | Must state the type, which can be FULL, CHROM or TABIX          |
++-------------------+-----------------------------------------------------------------+
+| ``-l <level>``    | Compression level (0-9). Default 1.                             |
++-------------------+-----------------------------------------------------------------+
+| ``-t '<tags>'``   | List of tags which write ensures a file will be created.        |
+|                   | Only valid with the -f option.                                  |
++-------------------+-----------------------------------------------------------------+
+| ``-tags '<tags>'``| List of tags/alias to use in the resulting dictionary when      |
+|                   | writing the files to directories.   Usually used with partgor   |
+|                   | as ``-tags #{tags}``.                                           |
++-------------------+-----------------------------------------------------------------+
+| ``-prefix <hf>``  | Takes in a text source containing prefix to be prepended to the |
+|                   | file written. Also support string in single quotes              |
++-------------------+-----------------------------------------------------------------+
+| ``-noheader``     | Don't write a header lines.  Not valid with gor/gorz/nor/norz.  |
++-------------------+-----------------------------------------------------------------+
+| ``-card '<cols>'``| Calculate cardinality of columns in '<cols>' and adds to the    |
+|                   | outputs meta data.                                              |
++-------------------+-----------------------------------------------------------------+
+| ``-link <link>``  | Writes a link file pointing to the the <file name>.             |
+|                   |                                                                 |
++-------------------+-----------------------------------------------------------------+
+| ``-vlink <link>`` | Writes a versioned link file pointing to the the <file name>.   |
+|                   | The <file name> should not be overwritten if it has previously  |
+|                   | been used in a link file.                                       |
++-------------------+-----------------------------------------------------------------+
 
 Examples
 ========

documentation/src/commands.rst

@@ -270,8 +270,11 @@ Commands
      - Same as SELECT, but ignores errors generated from incorrect syntax.
      - GOR/NOR
    * - :ref:`TRYWHERE`
-     - Same as WHERE, but ignores errors generated from incorrect syntax.
+      - Same as WHERE, but ignores errors generated from incorrect syntax.
      - GOR/NOR
+   * - :ref:`TSVAPPEND`
+     - Similar as WRITE, but appends to the given file.
+     - NOR
    * - :ref:`UNPIVOT`
      - Takes information in multiple rows and splits them into multiple rows as attribute-value pairs.
      - GOR/NOR

drivers/src/main/java/org/gorpipe/s3/shared/S3ProjectSharedProjectSourceProvider.java

@@ -46,7 +46,7 @@ protected String getFallbackUrl(String url) {
 
     @Override
     protected void updateSharedSourceLink(S3SharedSource source, String project) {
-        source.setProjectLinkFile(DataUtil.toFile(PathUtils.stripTrailingSlash(getRelativePath(source)), DataType.LINK));
+        source.setProjectLinkFile(DataUtil.toLink(PathUtils.stripTrailingSlash(getRelativePath(source))));
 
         source.setProjectLinkFileContent(String.format("%sprojects/%s/%s",
                 S3ProjectSharedSourceType.PREFIX, project, getRelativePath(source)));

drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSource.java

@@ -26,9 +26,9 @@
 import org.apache.commons.lang3.StringUtils;
 import org.gorpipe.base.security.Credentials;
 import org.gorpipe.exceptions.GorResourceException;
-import org.gorpipe.gor.driver.meta.DataType;
 import org.gorpipe.gor.driver.meta.SourceReference;
 import org.gorpipe.gor.table.util.PathUtils;
+import org.gorpipe.gor.util.DataUtil;
 import org.gorpipe.s3.driver.*;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.services.s3.S3Client;
@@ -140,7 +140,7 @@ private String removeExtraFolder(String path) {
     public SourceReference getTopSourceReference() {
         // Shared source should be access though links, so find the first link (which should be the direct access link)
         SourceReference top = getSourceReference();
-        while (top.getParentSourceReference() != null && !top.getParentSourceReference().getUrl().endsWith(DataType.LINK.suffix)) {
+        while (top.getParentSourceReference() != null && !DataUtil.isLink(top.getParentSourceReference().getUrl())) {
             top = top.getParentSourceReference();
         }
         return top;

drivers/src/main/java/org/gorpipe/s3/shared/S3SharedSourceProvider.java

@@ -123,7 +123,7 @@ public S3SharedSource resolveDataSource(SourceReference sourceReference)
     }
 
     protected void updateSharedSourceLink(S3SharedSource source, String project) {
-        source.setProjectLinkFile(DataUtil.toFile(PathUtils.stripTrailingSlash(getRelativePath(source)), DataType.LINK));
+        source.setProjectLinkFile(DataUtil.toLink(getRelativePath(source)));
         source.setProjectLinkFileContent(findSharedSourceLinkContent(source));
     }
 
@@ -216,10 +216,7 @@ private String createErrorMessageForFailure(SourceReference sourceReference, S3S
     private SourceReference createFallbackSourceReference(SourceReference sourceReference) {
         String fallbackUrl = getFallbackUrl(sourceReference.getUrl());
         if (fallbackUrl != null) {
-            SourceReference updatedSourceReference = new SourceReference(fallbackUrl, sourceReference.securityContext, sourceReference.commonRoot,
-                    sourceReference.getLookup(), sourceReference.getLinkSubPath(),
-                    sourceReference.isWriteSource());
-            return updatedSourceReference;
+            return new SourceReference(fallbackUrl, sourceReference);
         }
 
         return null;

drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java

@@ -9,6 +9,7 @@
 import org.gorpipe.gor.driver.PluggableGorDriver;
 import org.gorpipe.gor.driver.meta.DataType;
 import org.gorpipe.gor.driver.meta.SourceReference;
+import org.gorpipe.gor.driver.meta.SourceReferenceBuilder;
 import org.gorpipe.gor.model.DriverBackedFileReader;
 import org.gorpipe.gor.model.DriverBackedSecureFileReader;
 import org.gorpipe.gor.model.FileReader;
@@ -19,7 +20,6 @@
 import org.gorpipe.utils.DriverUtils;
 import org.junit.*;
 import org.junit.contrib.java.lang.system.EnvironmentVariables;
-import org.junit.contrib.java.lang.system.ProvideSystemProperty;
 import org.junit.contrib.java.lang.system.RestoreSystemProperties;
 import org.junit.contrib.java.lang.system.SystemErrRule;
 import org.junit.experimental.categories.Category;
@@ -402,7 +402,7 @@ public void testProjectWriteUserDataWithIndexServer() throws IOException {
             source.delete();
         }
 
-        Assert.assertTrue(Files.exists(Path.of(gorRoot, dataPath + ".link")));
+        Assert.assertTrue(Files.exists(Path.of(gorRoot, dataPath + DataType.LINK.suffix)));
     }
 
     @Test
@@ -554,7 +554,7 @@ public void testWriteExplicitWrite() throws IOException {
 
     private DataSource getDataSourceFromProvider(S3SharedSourceProvider provider, String relativePath,
                                                  Credentials.OwnerType ownerType, String owner) throws IOException {
-        SourceReference sourceReference = new SourceReference.Builder(provider.getSharedUrlPrefix() + relativePath)
+        SourceReference sourceReference = new SourceReferenceBuilder(provider.getSharedUrlPrefix() + relativePath)
                 .commonRoot("projects/some_project")
                 .securityContext(createSecurityContext(provider.getService(), ownerType, owner, S3_KEY, S3_SECRET))
                 .build();

drivers/src/test/java/org/gorpipe/s3/shared/UTestS3Shared.java

@@ -5,6 +5,7 @@
 import org.gorpipe.base.security.Credentials;
 import org.gorpipe.gor.driver.DataSource;
 import org.gorpipe.gor.driver.meta.SourceReference;
+import org.gorpipe.gor.driver.meta.SourceReferenceBuilder;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -198,7 +199,7 @@ public void testResolveSourceGlobaluseHighestTypeInLinksFalse() throws IOExcepti
 
     private DataSource getDataSourceFromProvider(S3SharedSourceProvider provider, String relativePath,
                                                  Credentials.OwnerType ownerType, String owner) throws IOException {
-        SourceReference sourceReference = new SourceReference.Builder(provider.getSharedUrlPrefix() + relativePath)
+        SourceReference sourceReference = new SourceReferenceBuilder(provider.getSharedUrlPrefix() + relativePath)
                 .commonRoot("projects/some_project")
                 .securityContext(createSecurityContext(provider.getService(), "some_s3_bucket", ownerType, owner))
                 .build();

gorscripts/src/test/java/org/gorpipe/gorshell/UTestHelpCmd.java

@@ -39,7 +39,7 @@ public class UTestHelpCmd {
             "GORROWS, GORSQL, GRANNO, GREP, GROUP, GTGEN, GTLD, HIDE, INSET, JOIN, LDANNO, LEFTJOIN, LEFTWHERE, LIFTOVER, LOG, MAP, MERGE,",
             "MULTIMAP, NOR, NORCMD, NORIF, NORROWS, NORSQL, PARALLEL, PARTGOR, PEDPIVOT, PGOR, PILEUP, PIPESTEPS, PIVOT, PLINKREGRESSION, PREFIX,",
             "RANK, REGRESSION, REGSEL, RENAME, REPLACE, ROWNUM, SDL, SED, SEGHIST, SEGPROJ, SEGSPAN, SELECT, SEQ, SETCOLTYPE, SIGNATURE, SKIP,",
-            "SORT, SPLIT, SQL, TEE, THROWIF, TOGOR, TOP, TRYHIDE, TRYSELECT, TRYWHERE, UNPIVOT, UNTIL, VARGROUP, VARIANTS, VARJOIN, VARMERGE,",
+            "SORT, SPLIT, SQL, TEE, THROWIF, TOGOR, TOP, TRYHIDE, TRYSELECT, TRYWHERE, TSVAPPEND, UNPIVOT, UNTIL, VARGROUP, VARIANTS, VARJOIN, VARMERGE,",
             "VARNORM, VERIFYCOLTYPE, VERIFYORDER, WAIT, WHERE, WRITE",
             "",
             "->ADJUST",

gortools/src/main/java/gorsat/process/GorJavaUtilities.java

@@ -614,9 +614,9 @@ public static void writeDictionaryFromMeta(FileReader fileReader, String outfold
                     dictionarypathwriter.write(entry);
                 }
             } else writeDummyHeader(dictionarypathwriter);
-
-            localFileReader.writeLinkIfNeeded(dictionarypath);
         }
+
+        localFileReader.writeLinkIfNeeded(dictionarypath);
     }
 
     public static Optional<String[]> parseDictionaryColumn(String[] dictList, FileReader fileReader) {

gortools/src/main/java/org/gorpipe/gor/manager/BucketManager.java

@@ -589,7 +589,7 @@ private void deleteFileIfExists(String path) {
 
     private void deleteLinkFileIfExists(String path) {
         try {
-            String linkFile = DataUtil.isLink(path) ? path : DataUtil.toFile(path, DataType.LINK);
+            String linkFile = DataUtil.isLink(path) ? path : DataUtil.toLink(path);
             DataSource linkSource = table.getFileReader().resolveDataSource(table.getFileReader().createSourceReference(linkFile, false));
             if (linkSource != null && linkSource.exists()) {
                 linkSource.delete();
@@ -715,7 +715,7 @@ private boolean containsBucketFile(String bucketFile, Set<String> usedBuckets) {
         if (usedBuckets.contains(bucketFile)) {
             return true;
         }
-        if (bucketFile.endsWith(".link")) {
+        if (DataUtil.isLink(bucketFile)) {
             String bucketFileNoLink = bucketFile.substring(0, bucketFile.length() - 5);
             if (usedBuckets.contains(bucketFileNoLink)) {
                 return true;