Groonga - fulltext search engine.

[Groonga-commit] groonga/grnxx at d73b0b9 [master] Gnx: implement Load() and LoadMap().

susumu.yata null+****@clear*****
Wed Apr 1 12:44:39 JST 2015

• Previous message: [Groonga-commit] droonga/wikipedia-search.wiki at b843f4d [master] Updated ノンストップでノードを追加する場面の検証手順 (markdown)
• Next message: [Groonga-commit] droonga/wikipedia-search.wiki at 34865eb [master] Updated ノンストップでノードを追加する場面の検証手順 (markdown)
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

susumu.yata	2015-04-01 12:44:39 +0900 (Wed, 01 Apr 2015)

  New Revision: d73b0b97e8997a360f87a7d6d8e70ef6e9ee18e3
  https://github.com/groonga/grnxx/commit/d73b0b97e8997a360f87a7d6d8e70ef6e9ee18e3

  Message:
    Gnx: implement Load() and LoadMap().

  Added files:
    go2/.gitignore
  Modified files:
    go2/gnx/gnx.go
    go2/gnxTest.go

  Added: go2/.gitignore (+1 -0) 100644
===================================================================
--- /dev/null
+++ go2/.gitignore    2015-04-01 12:44:39 +0900 (8a78dfc)
@@ -0,0 +1 @@
+gnxTest

  Modified: go2/gnx/gnx.go (+172 -13)
===================================================================
--- go2/gnx/gnx.go    2015-04-01 08:37:07 +0900 (dc16fca)
+++ go2/gnx/gnx.go    2015-04-01 12:44:39 +0900 (ba57563)
@@ -8,10 +8,13 @@ package gnx
 import "C"
 
 import (
+	"encoding/binary"
 	"encoding/json" // FIXME
 	"fmt"
+	"hash/fnv"
 	"io/ioutil"
 	"math"
+	"math/rand"
 	"os"
 	"strconv"
 	"strings"
@@ -326,8 +329,8 @@ func (db *GroongaDB) loadC(
 				numValues, thisLen)
 		}
 	}
-	if numValues == -1 {
-		numValues = 0
+	if numValues <= 0 {
+		return 0, fmt.Errorf("no input")
 	}
 
 	// TODO: Use cgo for JSON encoding.
@@ -482,22 +485,166 @@ func (db *DB) checkColumnName(columnName string) error {
 	return nil
 }
 
-func (db *DB) loadN(
+func (db *DB) load(
 	tableName string, columnNames []string, records [][]Valuer) (int, error) {
-	return 0, nil
+	idID := -1
+	keyID := -1
+	for i, columnName := range columnNames {
+		switch columnName {
+		case "_id":
+			if idID != -1 {
+				return 0, fmt.Errorf("_id appears more than once")
+			}
+			idID = i
+		case "_key":
+			if keyID != -1 {
+				return 0, fmt.Errorf("_key appears more than once")
+			}
+			keyID = i
+		}
+	}
+	if (idID != -1) && (keyID != -1) {
+		return 0, fmt.Errorf("both _id and _key appear")
+	}
+
+	recordsPerDBs := make([][][]Valuer, len(db.groongaDBs))
+	switch {
+	case idID != -1:
+		for _, record := range records {
+			rowID := record[idID].(Int)
+			dbID := int(rowID - 1) % len(db.groongaDBs)
+			dummyRecord := make([]Valuer, len(record))
+			copy(dummyRecord, record)
+			dummyRecord[idID] = Int((int(rowID - 1) / len(db.groongaDBs)) + 1)
+			recordsPerDBs[dbID] = append(recordsPerDBs[dbID], dummyRecord)
+		}
+	case keyID != -1:
+		for _, record := range records {
+			switch key := record[keyID].(type) {
+			case Int:
+				hasher := fnv.New32a()
+				err := binary.Write(hasher, binary.LittleEndian, key)
+				if err != nil {
+					return 0, err
+				}
+				dbID := int(hasher.Sum32()) % len(db.groongaDBs)
+				recordsPerDBs[dbID] = append(recordsPerDBs[dbID], record)
+			case Float:
+				hasher := fnv.New32a()
+				err := binary.Write(hasher, binary.LittleEndian, key)
+				if err != nil {
+					return 0, err
+				}
+				dbID := int(hasher.Sum32()) % len(db.groongaDBs)
+				recordsPerDBs[dbID] = append(recordsPerDBs[dbID], record)
+			case Text:
+				hasher := fnv.New32a()
+				if _, err := hasher.Write([]byte(key)); err != nil {
+					return 0, err
+				}
+				dbID := int(hasher.Sum32()) % len(db.groongaDBs)
+				recordsPerDBs[dbID] = append(recordsPerDBs[dbID], record)
+			default:
+				return 0, fmt.Errorf("unsupported key type")
+			}
+		}
+	default:
+		for _, record := range records {
+			dbID := rand.Int() % len(db.groongaDBs)
+			recordsPerDBs[dbID] = append(recordsPerDBs[dbID], record)
+		}
+	}
+
+	// TODO: Parallel processing.
+	total := 0
+	for i, recordsPerDB := range recordsPerDBs {
+		if len(recordsPerDB) != 0 {
+			count, err := db.groongaDBs[i].load(tableName, columnNames, recordsPerDB)
+			if err != nil {
+				return total, err
+			}
+			total += count
+		}
+	}
+	return total, nil
 }
 
-func (db *DB) loadMapN(
-	tableName string, records []map[string]Valuer) (int, error) {
-	return 0, nil
+func (db *DB) loadMap(
+	tableName string, recordMaps []map[string]Valuer) (int, error) {
+	recordMapsPerDBs := make([][]map[string]Valuer, len(db.groongaDBs))
+	for _, recordMap := range recordMaps {
+		rowIDValue, hasRowID := recordMap["_id"]
+		keyValue, hasKey := recordMap["_key"]
+		if hasRowID && hasKey {
+			return 0, fmt.Errorf("both _id and _key appear")
+		}
+		switch {
+		case hasRowID:
+			rowID := rowIDValue.(Int)
+			dbID := int(rowID - 1) % len(db.groongaDBs)
+			dummyRecordMap := make(map[string]Valuer)
+			for key, value := range recordMap {
+				if key == "_id" {
+					dummyRecordMap[key] = Int((int(rowID - 1) / len(db.groongaDBs)) + 1)
+				} else {
+					dummyRecordMap[key] = value
+				}
+			}
+			recordMapsPerDBs[dbID] = append(recordMapsPerDBs[dbID], dummyRecordMap)
+		case hasKey:
+			switch key := keyValue.(type) {
+			case Int:
+				hasher := fnv.New32a()
+				err := binary.Write(hasher, binary.LittleEndian, key)
+				if err != nil {
+					return 0, err
+				}
+				dbID := int(hasher.Sum32()) % len(db.groongaDBs)
+				recordMapsPerDBs[dbID] = append(recordMapsPerDBs[dbID], recordMap)
+			case Float:
+				hasher := fnv.New32a()
+				err := binary.Write(hasher, binary.LittleEndian, key)
+				if err != nil {
+					return 0, err
+				}
+				dbID := int(hasher.Sum32()) % len(db.groongaDBs)
+				recordMapsPerDBs[dbID] = append(recordMapsPerDBs[dbID], recordMap)
+			case Text:
+				hasher := fnv.New32a()
+				if _, err := hasher.Write([]byte(key)); err != nil {
+					return 0, err
+				}
+				dbID := int(hasher.Sum32()) % len(db.groongaDBs)
+				recordMapsPerDBs[dbID] = append(recordMapsPerDBs[dbID], recordMap)
+			default:
+				return 0, fmt.Errorf("unsupported key type")
+			}
+		default:
+			dbID := rand.Int() % len(db.groongaDBs)
+			recordMapsPerDBs[dbID] = append(recordMapsPerDBs[dbID], recordMap)
+		}
+	}
+
+	// TODO: Parallel processing.
+	total := 0
+	for i, recordMapsPerDB := range recordMapsPerDBs {
+		if len(recordMapsPerDB) != 0 {
+			count, err := db.groongaDBs[i].loadMap(tableName, recordMapsPerDB)
+			if err != nil {
+				return total, err
+			}
+			total += count
+		}
+	}
+	return total, nil
 }
 
-func (db *DB) loadCN(
+func (db *DB) loadC(
 	tableName string, columnNames []string, records []interface{}) (int, error) {
 	return 0, nil
 }
 
-func (db *DB) loadCMapN(
+func (db *DB) loadCMap(
 	tableName string, records map[string]interface{}) (int, error) {
 	return 0, nil
 }
@@ -508,6 +655,9 @@ func (db *DB) Load(
 	if err := db.checkTableName(tableName); err != nil {
 		return 0, err
 	}
+	if (len(columnNames) == 0) || (len(records) == 0) {
+		return 0, fmt.Errorf("no input")
+	}
 	for _, columnName := range columnNames {
 		if err := db.checkColumnName(columnName); err != nil {
 			return 0, err
@@ -524,7 +674,7 @@ func (db *DB) Load(
 	if len(db.groongaDBs) == 1 {
 		return db.groongaDBs[0].load(tableName, columnNames, records)
 	} else {
-		return db.loadN(tableName, columnNames, records)
+		return db.load(tableName, columnNames, records)
 	}
 }
 
@@ -534,6 +684,9 @@ func (db *DB) LoadMap(
 	if err := db.checkTableName(tableName); err != nil {
 		return 0, err
 	}
+	if len(recordMaps) == 0 {
+		return 0, fmt.Errorf("no input")
+	}
 	for _, recordMap := range recordMaps {
 		for columnName, _ := range recordMap {
 			if err := db.checkColumnName(columnName); err != nil {
@@ -545,7 +698,7 @@ func (db *DB) LoadMap(
 	if len(db.groongaDBs) == 1 {
 		return db.groongaDBs[0].loadMap(tableName, recordMaps)
 	} else {
-		return db.loadMapN(tableName, recordMaps)
+		return db.loadMap(tableName, recordMaps)
 	}
 }
 
@@ -556,6 +709,9 @@ func (db *DB) LoadC(
 	if err := db.checkTableName(tableName); err != nil {
 		return 0, err
 	}
+	if len(columnNames) == 0 {
+		return 0, fmt.Errorf("no input")
+	}
 	for _, columnName := range columnNames {
 		if err := db.checkColumnName(columnName); err != nil {
 			return 0, err
@@ -570,7 +726,7 @@ func (db *DB) LoadC(
 	if len(db.groongaDBs) == 1 {
 		return db.groongaDBs[0].loadC(tableName, columnNames, columnarRecords)
 	} else {
-		return db.loadCN(tableName, columnNames, columnarRecords)
+		return db.loadC(tableName, columnNames, columnarRecords)
 	}
 }
 
@@ -580,6 +736,9 @@ func (db *DB) LoadCMap(
 	if err := db.checkTableName(tableName); err != nil {
 		return 0, err
 	}
+	if len(columnarRecordsMap) == 0 {
+		return 0, fmt.Errorf("no input")
+	}
 	for columnName, _ := range columnarRecordsMap {
 		if err := db.checkColumnName(columnName); err != nil {
 			return 0, err
@@ -589,6 +748,6 @@ func (db *DB) LoadCMap(
 	if len(db.groongaDBs) == 1 {
 		return db.groongaDBs[0].loadCMap(tableName, columnarRecordsMap)
 	} else {
-		return db.loadCMapN(tableName, columnarRecordsMap)
+		return db.loadCMap(tableName, columnarRecordsMap)
 	}
 }

  Modified: go2/gnxTest.go (+158 -12)
===================================================================
--- go2/gnxTest.go    2015-04-01 08:37:07 +0900 (ee2ae2a)
+++ go2/gnxTest.go    2015-04-01 12:44:39 +0900 (4a51861)
@@ -5,7 +5,9 @@ import "fmt"
 import "log"
 import "os"
 
-func main() {
+func testA() {
+	log.Println("testA()")
+
 	db, dir, err := gnx.CreateTempDB("", "gnxConsole", 1)
 	if err != nil {
 		log.Println(err)
@@ -27,10 +29,10 @@ func main() {
 	}
 
 	{
-		records := make([][]gnx.Valuer, 3)
-		records[0] = []gnx.Valuer{gnx.Int(10)}
-		records[1] = []gnx.Valuer{gnx.Int(20)}
-		records[2] = []gnx.Valuer{gnx.Int(30)}
+		var records [][]gnx.Valuer
+		records = append(records, []gnx.Valuer{gnx.Int(1)})
+		records = append(records, []gnx.Valuer{gnx.Int(2)})
+		records = append(records, []gnx.Valuer{gnx.Int(3)})
 		count, err := db.Load("Table", []string{"Value"}, records)
 		if err != nil {
 			log.Println(err)
@@ -40,10 +42,10 @@ func main() {
 	}
 
 	{
-		recordMaps := make([]map[string]gnx.Valuer, 3)
-		recordMaps[0] = map[string]gnx.Valuer{"Value":gnx.Int(100)}
-		recordMaps[1] = map[string]gnx.Valuer{"Value":gnx.Int(200)}
-		recordMaps[2] = map[string]gnx.Valuer{"Value":gnx.Int(300)}
+		var recordMaps []map[string]gnx.Valuer
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{"Value":gnx.Int(10)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{"Value":gnx.Int(20)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{"Value":gnx.Int(30)})
 		count, err := db.LoadMap("Table", recordMaps)
 		if err != nil {
 			log.Println(err)
@@ -53,8 +55,8 @@ func main() {
 	}
 
 	{
-		columnarRecords := make([]interface{}, 1)
-		columnarRecords[0] = []gnx.Int{-10,-20,-30}
+		var columnarRecords []interface{}
+		columnarRecords = append(columnarRecords, []gnx.Int{-1,-2,-3})
 		count, err := db.LoadC("Table", []string{"Value"}, columnarRecords)
 		if err != nil {
 			log.Println(err)
@@ -65,7 +67,7 @@ func main() {
 
 	{
 		columnarRecordsMap := make(map[string]interface{})
-		columnarRecordsMap["Value"] = []gnx.Int{-100,-200,-300}
+		columnarRecordsMap["Value"] = []gnx.Int{-10,-20,-30}
 		count, err := db.LoadCMap("Table", columnarRecordsMap)
 		if err != nil {
 			log.Println(err)
@@ -82,3 +84,147 @@ func main() {
 	}
 	fmt.Println("result:", string(jsonBytes))
 }
+
+func testB() {
+	log.Println("testB()")
+
+	db, dir, err := gnx.CreateTempDB("", "gnxConsole", 3)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	defer os.RemoveAll(dir)
+	defer db.Close()
+
+	for i := 0; i < 3; i++ {
+		_, err = db.GroongaQuery(i, "table_create Table TABLE_NO_KEY")
+		if err != nil {
+			log.Println(err)
+			return
+		}
+	}
+
+	for i := 0; i < 3; i++ {
+		_, err = db.GroongaQuery(i, "column_create Table Value COLUMN_SCALAR Int32")
+			if err != nil {
+			log.Println(err)
+			return
+		}
+	}
+
+	{
+		var records [][]gnx.Valuer
+		records = append(records, []gnx.Valuer{gnx.Int(1)})
+		records = append(records, []gnx.Valuer{gnx.Int(2)})
+		records = append(records, []gnx.Valuer{gnx.Int(3)})
+		records = append(records, []gnx.Valuer{gnx.Int(4)})
+		count, err := db.Load("Table", []string{"Value"}, records)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		fmt.Println("count:", count)
+	}
+
+	{
+		var records [][]gnx.Valuer
+		records = append(records, []gnx.Valuer{gnx.Int(1), gnx.Int(5)})
+		records = append(records, []gnx.Valuer{gnx.Int(2), gnx.Int(6)})
+		records = append(records, []gnx.Valuer{gnx.Int(3), gnx.Int(7)})
+		records = append(records, []gnx.Valuer{gnx.Int(3), gnx.Int(8)})
+		fmt.Println("records (before):", records)
+		count, err := db.Load("Table", []string{"_id","Value"}, records)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		fmt.Println("records (after):", records)
+		fmt.Println("count:", count)
+	}
+
+	{
+		var recordMaps []map[string]gnx.Valuer
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{"Value":gnx.Int(10)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{"Value":gnx.Int(20)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{"Value":gnx.Int(30)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{"Value":gnx.Int(40)})
+		fmt.Println("recordMaps (before):", recordMaps)
+		count, err := db.LoadMap("Table", recordMaps)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		fmt.Println("recordMaps (after):", recordMaps)
+		fmt.Println("count:", count)
+	}
+
+	{
+		var recordMaps []map[string]gnx.Valuer
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{
+			"_id":gnx.Int(1), "Value":gnx.Int(50)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{
+			"_id":gnx.Int(2), "Value":gnx.Int(60)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{
+			"_id":gnx.Int(3), "Value":gnx.Int(70)})
+		recordMaps = append(recordMaps, map[string]gnx.Valuer{
+			"_id":gnx.Int(4), "Value":gnx.Int(80)})
+		fmt.Println("recordMaps (before):", recordMaps)
+		count, err := db.LoadMap("Table", recordMaps)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		fmt.Println("recordMaps (after):", recordMaps)
+		fmt.Println("count:", count)
+	}
+
+//	{
+//		recordMaps := make([]map[string]gnx.Valuer, 3)
+//		recordMaps[0] = map[string]gnx.Valuer{"Value":gnx.Int(100)}
+//		recordMaps[1] = map[string]gnx.Valuer{"Value":gnx.Int(200)}
+//		recordMaps[2] = map[string]gnx.Valuer{"Value":gnx.Int(300)}
+//		count, err := db.LoadMap("Table", recordMaps)
+//		if err != nil {
+//			log.Println(err)
+//			return
+//		}
+//		fmt.Println("count:", count)
+//	}
+
+//	{
+//		columnarRecords := make([]interface{}, 1)
+//		columnarRecords[0] = []gnx.Int{-10,-20,-30}
+//		count, err := db.LoadC("Table", []string{"Value"}, columnarRecords)
+//		if err != nil {
+//			log.Println(err)
+//			return
+//		}
+//		fmt.Println("count:", count)
+//	}
+
+//	{
+//		columnarRecordsMap := make(map[string]interface{})
+//		columnarRecordsMap["Value"] = []gnx.Int{-100,-200,-300}
+//		count, err := db.LoadCMap("Table", columnarRecordsMap)
+//		if err != nil {
+//			log.Println(err)
+//			return
+//		}
+//		fmt.Println("count:", count)
+//	}
+
+	command := "select Table --limit -1"
+	for i := 0; i < 3; i++ {
+		jsonBytes, err := db.GroongaQuery(i, command)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		fmt.Printf("result[%d]: %s\n", i, string(jsonBytes))
+	}
+}
+
+func main() {
+	testA()
+	testB()
+}
-------------- next part --------------
HTML����������������������������...
다운로드 

• Previous message: [Groonga-commit] droonga/wikipedia-search.wiki at b843f4d [master] Updated ノンストップでノードを追加する場面の検証手順 (markdown)
• Next message: [Groonga-commit] droonga/wikipedia-search.wiki at 34865eb [master] Updated ノンストップでノードを追加する場面の検証手順 (markdown)
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]