Browse Source

增加repWrite部分TODO

gitlink
17BaoH 2 years ago
parent
commit
d42fca161f
13 changed files with 46 additions and 60 deletions
  1. BIN
      assets/test
  2. BIN
      assets/test1
  3. +0
    -1
      assets/test2
  4. +0
    -1
      assets/test3
  5. +0
    -0
      assets/testin
  6. +0
    -0
      assets/testin2
  7. BIN
      client
  8. +31
    -7
      clientCommand.go
  9. +1
    -23
      go.mod
  10. +0
    -22
      go.sum
  11. +14
    -4
      main.go
  12. +0
    -1
      test
  13. +0
    -1
      test1

BIN
assets/test View File


BIN
assets/test1 View File


+ 0
- 1
assets/test2 View File

@@ -1 +0,0 @@
滥忍性剀噤滥忍性剀噤tbdfhjlnprtbdfhjlnpr

+ 0
- 1
assets/test3 View File

@@ -1 +0,0 @@
122345678901234567890

+ 0
- 0
assets/testin View File


+ 0
- 0
assets/testin2 View File


BIN
client View File


+ 31
- 7
clientCommand.go View File

@@ -213,7 +213,11 @@ func repRead(fileSizeInBytes int64, ip string, repHash string, localFilePath str
if err != nil {
return
}

/*
TO DO: 判断本地有没有ipfs daemon、能否获取相应对象的cid
如果本地有ipfs daemon且能获取相应对象的cid,则获取对象cid对应的ipfsblock的cid,通过ipfs网络获取这些ipfsblock
否则,像目前一样,使用grpc向指定节点获取
*/
stream, _ := client.GetBlockOrReplica(context.Background(), &agentcaller.GetReq{
BlockOrReplicaHash: repHash,
})
@@ -235,7 +239,7 @@ func RepWrite(localFilePath string, bucketName string, objectName string, numRep
fileSizeInBytes := fileInfo.Size()
fmt.Println(fileSizeInBytes)

//计算每个块的packet数
//写入对象的packet数
numWholePacket := fileSizeInBytes / packetSizeInBytes
lastPacketInBytes := fileSizeInBytes % packetSizeInBytes
numPacket := numWholePacket
@@ -243,7 +247,8 @@ func RepWrite(localFilePath string, bucketName string, objectName string, numRep
numPacket++
}

//发送写请求,分配写入节点Ip
//发送写请求,请求Coor分配写入节点Ip
//TO DO: 加入两个字段,本机IP和当前进程号
command1 := rabbitmq.RepWriteCommand{
BucketName: bucketName,
ObjectName: objectName,
@@ -257,9 +262,15 @@ func RepWrite(localFilePath string, bucketName string, objectName string, numRep
rabbit1 := rabbitmq.NewRabbitMQSimple("coorQueue")
rabbit1.PublishSimple(b1)

//接收消息,赋值给ips
var res1 rabbitmq.WriteRes
var ips []string
/*
TODO xh: 判断writeRes里的状态码
如果有错,就报错返回,结束程序
如果没错,就把得到的IP值赋给ips
*/

//TODO xh: queueName调整:coorClientQueue+"_"+"本机Ip"+"_"+"进程号"
queueName := "coorClientQueue" + strconv.Itoa(userId)
rabbit2 := rabbitmq.NewRabbitMQSimple(queueName)
msgs := rabbit2.ConsumeSimple(time.Millisecond, true)
@@ -269,8 +280,8 @@ func RepWrite(localFilePath string, bucketName string, objectName string, numRep
for d := range msgs {
_ = json.Unmarshal(d.Body, &res1)
ips = res1.Ips
wg.Done()
}
wg.Done()
}()
wg.Wait()

@@ -285,11 +296,13 @@ func RepWrite(localFilePath string, bucketName string, objectName string, numRep
go loadDistribute(localFilePath, loadDistributeBufs[:], numWholePacket, lastPacketInBytes) //从本地文件系统加载数据
wg.Add(numRep)
for i := 0; i < numRep; i++ {
//TODO xh: send的第一个参数不需要了
go send("rep.json"+strconv.Itoa(i), ips[i], loadDistributeBufs[i], numPacket, &wg, hashs, i) //"block1.json"这样参数不需要
}
wg.Wait()

//第二轮通讯:插入元数据hashs
//TODO xh: 加入pid字段
command2 := rabbitmq.WriteHashCommand{
BucketName: bucketName,
ObjectName: objectName,
@@ -311,6 +324,7 @@ func RepWrite(localFilePath string, bucketName string, objectName string, numRep
if res2.MetaCode == 0 {
wg.Done()
}
//TODO xh: MetaCode不为零,代表插入出错,需输出错误
}
}()
wg.Wait()
@@ -470,11 +484,11 @@ func EcWrite(localFilePath string, bucketName string, objectName string, ecName
}

func repMove(ip string, hash string) {
//通过消息队列发送调度命令
//TO DO: 通过消息队列发送调度命令
}

func ecMove(ip string, hashs []string, ids []int, ecName string) {
//通过消息队列发送调度命令
//TO DO: 通过消息队列发送调度命令
}

func loadDistribute(localFilePath string, loadDistributeBufs []chan []byte, numWholePacket int64, lastPacketInBytes int64) {
@@ -609,6 +623,11 @@ func decode(inBufs []chan []byte, outBufs []chan []byte, blockSeq []int, ecK int

func send(blockName string, ip string, inBuf chan []byte, numPacket int64, wg *sync.WaitGroup, hashs []string, idx int) {
fmt.Println("send " + blockName)
/*
TO DO ss: 判断本地有没有ipfs daemon、能否与目标agent的ipfs daemon连通、本地ipfs目录空间是否充足
如果本地有ipfs daemon、能与目标agent的ipfs daemon连通、本地ipfs目录空间充足,将所有内容写入本地ipfs目录,得到对象的cid,发送cid给目标agent让其pin相应的对象
否则,像目前一样,使用grpc向指定节点获取
*/
//rpc相关
conn, err := grpc.Dial(ip+port, grpc.WithInsecure())
if err != nil {
@@ -647,6 +666,11 @@ func get(blockHash string, ip string, getBuf chan []byte, numPacket int64) {
if err != nil {
panic(err)
}
/*
TO DO: 判断本地有没有ipfs daemon、能否获取相应对象的cid
如果本地有ipfs daemon且能获取相应编码块的cid,则获取编码块cid对应的ipfsblock的cid,通过ipfs网络获取这些ipfsblock
否则,像目前一样,使用grpc向指定节点获取
*/
client := agentcaller.NewTranBlockOrReplicaClient(conn)
//rpc get
stream, _ := client.GetBlockOrReplica(context.Background(), &agentcaller.GetReq{


+ 1
- 23
go.mod View File

@@ -2,34 +2,12 @@ module client

go 1.18

require proto v0.0.0

replace proto => ../proto

replace rabbitmq => ../rabbitmq

replace ec => ../ec

replace utils => ../utils

require (
utils v0.0.0-00010101000000-000000000000
ec v0.0.0-00010101000000-000000000000
google.golang.org/grpc v1.53.0
rabbitmq v0.0.0
)
require google.golang.org/grpc v1.53.0

require (
github.com/beevik/etree v1.1.0 // indirect
github.com/go-ping/ping v1.1.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/klauspost/reedsolomon v1.11.7 // indirect
github.com/streadway/amqp v1.0.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect


+ 0
- 22
go.sum View File

@@ -1,37 +1,15 @@
github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0=
github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU=
github.com/klauspost/reedsolomon v1.11.7/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A=
github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=
github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=


+ 14
- 4
main.go View File

@@ -20,26 +20,36 @@ import (
)

func main() {
//TODO xh:加载配置文件,获取packetSizeInBytes、grpc port、ipfs port、最大副本数、本机公网Ip等信息,参照src/utils/config.go

args := os.Args
arg_num := len(os.Args)
for i := 0; i < arg_num; i++ {
fmt.Println(args[i])
}
//TODO: 改为交互式client,输入用户名及秘钥后进入交互界面
switch args[1] {
case "ecWrite":
EcWrite(args[2], args[3], args[4], args[5])
//TODO: 写入对象时,Coor判断对象是否已存在,如果存在,则直接返回
case "write":
numRep, _ := strconv.Atoi(args[5])
if numRep <= 0 || numRep > 10 {
print("write::InputError!")
if numRep <= 0 || numRep > 10 { //TODO xh:10改为从配置文件中读出的最大副本数
print("write::InputError!") //TODO xh:优化提示语
} else {
RepWrite(args[2], args[3], args[4], numRep)
}

case "read":
Read(args[2], args[3], args[4])
case "move":
Move(args[2], args[3], args[4]) //bucket object destination
}
/*
TO DO:
1. ls命令,显示用户指定桶下的所有对象,及相关的元数据
2. rm命令,用户指定bucket和object名,执行删除操作
3. update命令,用户发起对象更新命令,查询元数据,判断对象的冗余方式,删除旧对象(unpin所有的副本或编码块),写入新对象
4. ipfsStat命令,查看本地有无ipfsdaemon,ipfs目录的使用率
5. ipfsFlush命令,unpin本地ipfs目录中的所有cid(block)
*/
}

+ 0
- 1
test View File

@@ -1 +0,0 @@
zkxx4567890123456789qazwsxedcr

+ 0
- 1
test1 View File

@@ -1 +0,0 @@
In the flood of darkness, hope is the light. It brings comfort, faith, and confidence. It gives us guidance when we are lost, and gives support when we are afraid. And the moment we give up hope, we give up our lives. The world we live in is disintegrating into a place of malice and hatred, where we need hope and find it harder. In this world of fear, hope to find better, but easier said than done, the more meaningful life of faith will make life

Loading…
Cancel
Save