use db instead of searchrawtransactions

db/dcrpg:

Rewrite GetAddressTransactionsRawWithSkip to use the pg db instead of
the searchrawtransactions RPC.

Remove InsightSearchRPCAddressTransactions, which was unused.

api/types:

Stop using chainjson.VinPrevOut.

Add Vin type that is similar to chainjson.Vin.  The VinPrevOut type is
not even emulated because it was almost useless: the amout was
already in the Vin struct, and the address of the prev out is not of
high importance when requesting address transactions.

Redefine AddressTxRaw with new Vin type.

work with dcrd RPC version 7.0.0 or 8.0.0

Author: chappjc

README.md

@@ -132,9 +132,9 @@ Always run the Current release or on the Current stable branch. Do not use `mast
 - [Go](https://golang.org) 1.18 or 1.19
 - [Node.js](https://nodejs.org/en/download/) 16.x or 18.x. Node.js is only used
   as a build tool, and is **not used at runtime**.
-- Running `dcrd` running with `--txindex --addrindex`, and synchronized to the
-  current best block on the network. On startup, dcrdata will verify that the
-  dcrd version is compatible.
+- Running `dcrd` running with `--txindex`, and synchronized to the current best
+  block on the network. On startup, dcrdata will verify that the dcrd version is
+  compatible.
 - PostgreSQL 11+
 
 ## Docker Support
@@ -367,7 +367,6 @@ dcrd.conf:
 
 ```ini
 txindex=1
-addrindex=1
 ```
 
 If these parameters are not set, dcrdata will be unable to retrieve transaction

api/types/apitypes.go

@@ -24,6 +24,8 @@ type TimeAPI struct {
 	S dbtypes.TimeDef
 }
 
+var _ fmt.Stringer = TimeAPI{}
+
 // String formats the time in a human-friendly layout.
 func (t TimeAPI) String() string {
 	return t.S.String()
@@ -34,6 +36,9 @@ func (t TimeAPI) UNIX() int64 {
 	return t.S.UNIX()
 }
 
+var _ json.Marshaler = (*TimeAPI)(nil)
+var _ json.Unmarshaler = (*TimeAPI)(nil)
+
 // MarshalJSON is set as the default marshalling function for TimeAPI struct.
 func (t *TimeAPI) MarshalJSON() ([]byte, error) {
 	return json.Marshal(t.S.UNIX())
@@ -83,6 +88,7 @@ type Tx struct {
 	Block         *BlockID `json:"block,omitempty"`
 }
 
+// Vin is an alias for dcrd's rpc/jsonrpc/types/v3.Vin type.
 type Vin = chainjson.Vin
 
 // TxShort models info about transaction TxID
@@ -191,7 +197,7 @@ type Vout struct {
 	N                   uint32       `json:"n"`
 	Version             uint16       `json:"version"`
 	ScriptPubKeyDecoded ScriptPubKey `json:"scriptPubKey"`
-	Spend               *TxInputID   `json:"spend,omitempty"`
+	Spend               *TxInputID   `json:"spend,omitempty"` // unused?
 }
 
 // TxInputID specifies a transaction input as hash:vin_index.
@@ -380,19 +386,104 @@ type Address struct {
 	Transactions []*AddressTxShort `json:"address_transactions"`
 }
 
+// ScriptSig models the signature script used to redeem a transaction output.
+// type ScriptSig struct {
+// 	Asm string `json:"asm,omitempty"`
+// 	Hex string `json:"hex,omitempty"`
+// }
+
+// VinShort describes a transaction input with limited detail, for the address
+// txn API endpoints. In particular, there is no ScriptSig or Sequence, and the
+// string fields for Coinbase, Stakebase, and TreasurySpend are just booleans.
+type VinShort struct {
+	Coinbase      bool    `json:"coinbase"`
+	Stakebase     bool    `json:"stakebase"`
+	Treasurybase  bool    `json:"treasurybase"`
+	TreasurySpend bool    `json:"treasuryspend"`
+	Txid          string  `json:"txid"`
+	Vout          uint32  `json:"vout"`
+	Tree          int8    `json:"tree"`
+	AmountIn      float64 `json:"amountin"`
+	BlockHeight   *uint32 `json:"blockheight,omitempty"`
+	BlockIndex    *uint32 `json:"blockindex,omitempty"`
+	// No ScriptSig or Sequence
+}
+
+// MarshalJSON is used to marshal a Vin to JSON with special handling for when
+// the these are generate coins (should be zero txid).
+func (v *VinShort) MarshalJSON() ([]byte, error) {
+	switch {
+	case v.Coinbase:
+		generated := struct {
+			Coinbase bool    `json:"coinbase"`
+			AmountIn float64 `json:"amountin"`
+		}{
+			Coinbase: true,
+			AmountIn: v.AmountIn,
+		}
+		return json.Marshal(generated)
+	case v.Stakebase:
+		generated := struct {
+			Stakebase bool    `json:"stakebase"`
+			AmountIn  float64 `json:"amountin"`
+		}{
+			Stakebase: true,
+			AmountIn:  v.AmountIn,
+		}
+		return json.Marshal(generated)
+	case v.Treasurybase:
+		generated := struct {
+			Treasurybase bool    `json:"treasurybase"`
+			AmountIn     float64 `json:"amountin"`
+		}{
+			Treasurybase: true,
+			AmountIn:     v.AmountIn,
+		}
+		return json.Marshal(generated)
+	case v.TreasurySpend:
+		generated := struct {
+			TreasurySpend bool    `json:"treasuryspend"`
+			AmountIn      float64 `json:"amountin"`
+		}{
+			TreasurySpend: true,
+			AmountIn:      v.AmountIn,
+		}
+		return json.Marshal(generated)
+
+	}
+
+	return json.Marshal(struct {
+		Txid        string  `json:"txid"`
+		Vout        uint32  `json:"vout"`
+		Tree        int8    `json:"tree"`
+		AmountIn    float64 `json:"amountin"`
+		BlockHeight *uint32 `json:"blockheight,omitempty"`
+		BlockIndex  *uint32 `json:"blockindex,omitempty"`
+	}{
+		Txid:        v.Txid,
+		Vout:        v.Vout,
+		Tree:        v.Tree,
+		AmountIn:    v.AmountIn,
+		BlockHeight: v.BlockHeight,
+		BlockIndex:  v.BlockIndex,
+	})
+}
+
 // AddressTxRaw is modeled from SearchRawTransactionsResult but with size in
-// place of hex
+// place of hex, and a limited vin structure.
 type AddressTxRaw struct {
-	Size          int32                  `json:"size"`
-	TxID          string                 `json:"txid"`
-	Version       int32                  `json:"version"`
-	Locktime      uint32                 `json:"locktime"`
-	Vin           []chainjson.VinPrevOut `json:"vin"`
-	Vout          []Vout                 `json:"vout"`
-	Confirmations int64                  `json:"confirmations"`
-	BlockHash     string                 `json:"blockhash"`
-	Time          TimeAPI                `json:"time,omitempty"`
-	Blocktime     TimeAPI                `json:"blocktime,omitempty"`
+	Size          int32      `json:"size"`
+	TxID          string     `json:"txid"`
+	Version       int32      `json:"version"`
+	Locktime      uint32     `json:"locktime"`
+	Type          int32      `json:"type"`
+	Vin           []VinShort `json:"vin"`
+	Vout          []Vout     `json:"vout"`
+	Confirmations int64      `json:"confirmations"`
+	BlockHash     string     `json:"blockhash,omitempty"`
+	Time          TimeAPI    `json:"time,omitempty"`      // for mempool txns?
+	Blocktime     *TimeAPI   `json:"blocktime,omitempty"` // vs mined?
+	// BlockHeight   int64                  `json:"blockheight"`
 }
 
 // AddressTxShort is a subset of AddressTxRaw with just the basic tx details
@@ -438,37 +529,6 @@ type TxRawWithTxType struct {
 	TxType string
 }
 
-// ScriptSig models the signature script used to redeem the origin transaction
-// as a JSON object (non-coinbase txns only)
-type ScriptSig struct {
-	Asm string `json:"asm,omitempty"`
-	Hex string `json:"hex,omitempty"`
-}
-
-// PrevOut represents previous output for an input Vin.
-type PrevOut struct {
-	Addresses []string `json:"addresses,omitempty"`
-	Value     float64  `json:"value"`
-}
-
-// VinPrevOut is like Vin except it includes PrevOut.  It is used by
-// searchrawtransaction
-type VinPrevOut struct {
-	Coinbase    string     `json:"coinbase"`
-	Stakebase   string     `json:"stakebase"`
-	Txid        string     `json:"txid"`
-	Vout        uint32     `json:"vout"`
-	Tree        int8       `json:"tree"`
-	AmountIn    *float64   `json:"amountin,omitempty"`
-	BlockHeight *uint32    `json:"blockheight,omitempty"`
-	BlockIndex  *uint32    `json:"blockindex,omitempty"`
-	ScriptSig   *ScriptSig `json:"scriptSig"`
-	PrevOut     *PrevOut   `json:"prevOut"`
-	Sequence    uint32     `json:"sequence"`
-}
-
-// end copy-paste from chainjson
-
 // Status indicates the state of the server. All fields are mutex protected and
 // and should be set with the getters and setters.
 type Status struct {

cmd/dcrdata/internal/api/apiroutes.go

@@ -624,6 +624,7 @@ func (c *appContext) setTrimmedTxSpends(tx *apitypes.TrimmedTx) error {
 	return c.setOutputSpends(tx.TxID, tx.Vout)
 }
 
+// getTransaction handles the /tx/{txid} API endpoint.
 func (c *appContext) getTransaction(w http.ResponseWriter, r *http.Request) {
 	// Look up any spending transactions for each output of this transaction
 	// when the client requests spends with the URL query ?spends=true.
@@ -811,6 +812,7 @@ func (c *appContext) getTxSwapsInfo(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, swapsInfo, m.GetIndentCtx(r))
 }
 
+// getTransactions handles the /txns POST API endpoint.
 func (c *appContext) getTransactions(w http.ResponseWriter, r *http.Request) {
 	// Look up any spending transactions for each output of this transaction
 	// when the client requests spends with the URL query ?spends=true.
@@ -1857,6 +1859,8 @@ func (c *appContext) getAddressTransactions(w http.ResponseWriter, r *http.Reque
 	writeJSON(w, txs, m.GetIndentCtx(r))
 }
 
+// getAddressTransactionsRaw handles the various /address/{addr}/.../raw API
+// endpoints.
 func (c *appContext) getAddressTransactionsRaw(w http.ResponseWriter, r *http.Request) {
 	addresses, err := m.GetAddressCtx(r, c.Params)
 	if err != nil || len(addresses) > 1 {
@@ -1869,14 +1873,13 @@ func (c *appContext) getAddressTransactionsRaw(w http.ResponseWriter, r *http.Re
 	skip := int64(m.GetMCtx(r))
 	if count <= 0 {
 		count = 10
-	} else if count > 8000 {
-		count = 8000
+	} else if count > 1000 {
+		count = 1000
 	}
 	if skip <= 0 {
 		skip = 0
 	}
 
-	// TODO: add postgresql powered method
 	txs := c.DataSource.GetAddressTransactionsRawWithSkip(address, int(count), int(skip))
 	if txs == nil {
 		http.Error(w, http.StatusText(422), 422)

cmd/dcrdata/internal/middleware/apimiddleware.go

@@ -766,6 +766,12 @@ func AddressPathCtxN(n int) func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			addressStr := chi.URLParam(r, "address")
 			if len(addressStr) < minAddressLength {
+				apiLog.Warnf("AddressPathCtxN rejecting address parameter of length %d", len(addressStr))
+				http.Error(w, "invalid address", http.StatusUnprocessableEntity)
+				return
+			}
+			if n == 1 && len(addressStr) > maxAddressLength {
+				apiLog.Warnf("AddressPathCtxN rejecting address parameter of length %d", len(addressStr))
 				http.Error(w, "invalid address", http.StatusUnprocessableEntity)
 				return
 			}

db/dbtypes/extraction.go

@@ -92,7 +92,7 @@ func processTransactions(msgBlock *wire.MsgBlock, tree int8, chainParams *chainc
 			BlockHash:        blockHash.String(),
 			BlockHeight:      int64(blockHeight),
 			BlockTime:        blockTime,
-			Time:             blockTime, // TODO, receive time?
+			Time:             blockTime, // TODO, receive time? no! REMOVE
 			TxType:           int16(txType),
 			Version:          tx.Version,
 			Tree:             tree,

db/dbtypes/types.go

@@ -1923,7 +1923,7 @@ type Tx struct {
 	BlockHash   string  `json:"block_hash"`
 	BlockHeight int64   `json:"block_height"`
 	BlockTime   TimeDef `json:"block_time"`
-	Time        TimeDef `json:"time"`
+	Time        TimeDef `json:"time"` // REMOVE!
 	TxType      int16   `json:"tx_type"`
 	Version     uint16  `json:"version"`
 	Tree        int8    `json:"tree"`

db/dcrpg/insightapi.go

@@ -131,29 +131,6 @@ func (pgb *ChainDB) AddressIDsByOutpoint(txHash string, voutIndex uint32) ([]uin
 	return ids, addrs, val, pgb.replaceCancelError(err)
 }
 
-// InsightSearchRPCAddressTransactions performs a searchrawtransactions for the
-// specified address, max number of transactions, and offset into the transaction
-// list. The search results are in reverse temporal order.
-// TODO: Does this really need all the prev vout extra data?
-func (pgb *ChainDB) InsightSearchRPCAddressTransactions(addr string, count,
-	skip int) []*chainjson.SearchRawTransactionsResult {
-	address, err := stdaddr.DecodeAddress(addr, pgb.chainParams)
-	if err != nil {
-		log.Infof("Invalid address %s: %v", addr, err)
-		return nil
-	}
-	prevVoutExtraData := true
-	ctx, cancel := context.WithTimeout(pgb.ctx, 10*time.Second)
-	defer cancel()
-	txs, err := pgb.Client.SearchRawTransactionsVerbose(ctx,
-		address, skip, count, prevVoutExtraData, true, nil)
-	if err != nil {
-		log.Warnf("GetAddressTransactions failed for address %s: %v", addr, err)
-		return nil
-	}
-	return txs
-}
-
 // GetTransactionHex returns the full serialized transaction for the specified
 // transaction hash as a hex encode string.
 func (pgb *ChainDB) GetTransactionHex(txid *chainhash.Hash) string {

db/dcrpg/internal/addrstmts.go

@@ -67,8 +67,8 @@ const (
 		` ON addresses(matching_tx_hash);`
 	DeindexAddressTableOnMatchingTxHash = `DROP INDEX IF EXISTS ` + IndexOfAddressTableOnMatchingTx + ` CASCADE;`
 
-	// TODO: figure out why this index exists since it's covered by the unique
-	// on tx_vin_vout_row_id too.
+	// IndexAddressTableOnAddress exists so address can be the first column in
+	// an index; it is second in tx_vin_vout_row_id.
 	IndexAddressTableOnAddress = `CREATE INDEX IF NOT EXISTS ` + IndexOfAddressTableOnAddress +
 		` ON addresses(address);`
 	DeindexAddressTableOnAddress = `DROP INDEX IF EXISTS ` + IndexOfAddressTableOnAddress + ` CASCADE;`
@@ -82,6 +82,59 @@ const (
 	// SelectFundingTxsByTx      = `SELECT id, prev_tx_hash FROM vins WHERE tx_hash=$1;`
 	// SelectFundingTxByTxIn     = `SELECT id, prev_tx_hash FROM vins WHERE tx_hash=$1 AND tx_index=$2;`
 
+	addressTxnsSubQuery = `SELECT tx_hash
+		FROM addresses
+		WHERE address = $1
+			AND valid_mainchain
+		GROUP BY tx_hash
+		ORDER BY MAX(block_time) DESC
+		LIMIT $2 OFFSET $3`
+
+	// need random table name? does lib/pq share sessions?
+	CreateTempAddrTxnsTable = `CREATE TEMPORARY TABLE address_transactions
+		ON COMMIT DROP -- do in a txn!
+		AS (` + addressTxnsSubQuery + `);`
+
+	SelectVinsForAddress0 = `SELECT vins.tx_hash, vins.tx_index, vins.prev_tx_hash, vins.prev_tx_index,
+			vins.prev_tx_tree, vins.value_in -- no block height or block index
+		FROM (` + addressTxnsSubQuery + `) atxs
+		-- JOIN transactions txs ON txs.tx_hash=atxs.tx_hash
+		-- JOIN vins ON vins.id = any(txs.vin_db_ids)
+		JOIN vins ON vins.tx_hash = atxs.tx_hash;`
+
+	SelectVinsForAddress = `SELECT vins.tx_hash, vins.tx_index, vins.prev_tx_hash, vins.prev_tx_index,
+			vins.prev_tx_tree, vins.value_in, prevtxs.block_height, prevtxs.block_index
+		FROM (` + addressTxnsSubQuery + `) atxs
+		JOIN vins ON vins.tx_hash = atxs.tx_hash   -- JOIN vins on vins.id = any(txs.vin_db_ids)
+		LEFT JOIN transactions prevtxs ON vins.prev_tx_hash=prevtxs.tx_hash;`  // LEFT JOIN because prev_tx_hash may be coinbase
+
+	SelectVoutsForAddress = `SELECT vouts.value, vouts.tx_hash, vouts.tx_index, vouts.version, vouts.pkscript
+		FROM (` + addressTxnsSubQuery + `) atxs
+		JOIN vouts ON vouts.tx_hash = atxs.tx_hash;`  //    -- vouts.id = any(transactions.vout_db_ids)
+
+	// select distinct tx_hash, block_time
+	// from addresses
+	// where address = 'DsSWTHFrsXV77SwAcMe451kJTwWjwPYjWTM' and valid_mainchain
+	// order by block_time desc
+	// limit 10 offset 0;
+
+	SelectAddressTxns = `SELECT txs.tx_hash, txs.block_hash, txs.block_height, txs.block_time,
+			txs.version, txs.lock_time, txs.size, txs.tx_type, cardinality(txs.vin_db_ids), cardinality(txs.vout_db_ids)
+		FROM (` + addressTxnsSubQuery + `) atxs
+		JOIN transactions txs ON txs.tx_hash = atxs.tx_hash
+		WHERE is_valid AND is_mainchain -- needed?
+		ORDER BY txs.block_time DESC;`
+
+	// SelectAddressTxnsAlt is very slow with a join on the full tables
+	SelectAddressTxnsAlt = `SELECT txs.tx_hash, txs.vin_db_ids, txs.vout_db_ids,
+			txs.block_hash, txs.block_height, txs.block_time,
+			txs.version, txs.lock_time, txs.size
+		FROM addresses
+		JOIN transactions txs ON addresses.tx_hash=txs.tx_hash AND valid_mainchain
+		WHERE address = $1 AND is_valid AND is_mainchain
+		ORDER BY block_height --  NOTE: height vs time
+		LIMIT $2 OFFSET $3;`
+
 	addrsColumnNames = `id, address, matching_tx_hash, tx_hash, tx_type, valid_mainchain,
 		tx_vin_vout_index, block_time, tx_vin_vout_row_id, value, is_funding`
 

db/dcrpg/internal/txstmts.go

@@ -17,7 +17,7 @@ const (
 		block_hash TEXT,
 		block_height INT8,
 		block_time TIMESTAMPTZ,
-		time TIMESTAMPTZ,
+		time TIMESTAMPTZ,  -- TODO: REMOVE!
 		tx_type INT4,
 		version INT4,
 		tree INT2,