提交 1cddf67a 作者: Jeromy

remove 'ipfs diag net' from codebase

License: MIT
Signed-off-by: 's avatarJeromy <jeromyj@gmail.com>
上级 a6e96e6c
......@@ -53,9 +53,6 @@ include $(dir)/Rules.mk
dir := exchange/bitswap/message/pb
include $(dir)/Rules.mk
dir := diagnostics/pb
include $(dir)/Rules.mk
dir := pin/internal/pb
include $(dir)/Rules.mk
......
package commands
import (
"bytes"
"errors"
"io"
"strings"
"text/template"
"time"
cmds "github.com/ipfs/go-ipfs/commands"
diag "github.com/ipfs/go-ipfs/diagnostics"
)
type DiagnosticConnection struct {
ID string
// TODO use milliseconds or microseconds for human readability
NanosecondsLatency uint64
Count int
}
var (
visD3 = "d3"
visDot = "dot"
visText = "text"
visFmts = []string{visD3, visDot, visText}
)
type DiagnosticPeer struct {
ID string
UptimeSeconds uint64
BandwidthBytesIn uint64
BandwidthBytesOut uint64
Connections []DiagnosticConnection
}
type DiagnosticOutput struct {
Peers []DiagnosticPeer
}
var DefaultDiagnosticTimeout = time.Second * 20
import cmds "github.com/ipfs/go-ipfs/commands"
var DiagCmd = &cmds.Command{
Helptext: cmds.HelpText{
......@@ -46,165 +8,7 @@ var DiagCmd = &cmds.Command{
},
Subcommands: map[string]*cmds.Command{
"net": diagNetCmd,
"sys": sysDiagCmd,
"cmds": ActiveReqsCmd,
},
}
var diagNetCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Generate a network diagnostics report.",
ShortDescription: `
Sends out a message to each node in the network recursively
requesting a listing of data about them including number of
connected peers and latencies between them.
The given timeout will be decremented 2s at every network hop,
ensuring peers try to return their diagnostics before the initiator's
timeout. If the timeout is too small, some peers may not be reached.
30s and 60s are reasonable timeout values, though networks vary.
The default timeout is 20 seconds.
The 'vis' option may be used to change the output format.
Three formats are supported:
* text - Easy to read. Default.
* d3 - json ready to be fed into d3view
* dot - graphviz format
The 'd3' format will output a json object ready to be consumed by
the chord network viewer, available at the following hash:
/ipfs/QmbesKpGyQGd5jtJFUGEB1ByPjNFpukhnKZDnkfxUiKn38
To view your diag output, 'ipfs add' the d3 vis output, and
open the following link:
http://gateway.ipfs.io/ipfs/QmbesKpGyQGd5jtJFUGEB1ByPjNFpukhnKZDnkfxUiKn38/chord#<your hash>
The 'dot' format can be fed into graphviz and other programs
that consume the dot format to generate graphs of the network.
`,
},
Options: []cmds.Option{
cmds.StringOption("vis", "Output format. One of: "+strings.Join(visFmts, ", ")).Default(visText),
},
Run: func(req cmds.Request, res cmds.Response) {
n, err := req.InvocContext().GetNode()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
if !n.OnlineMode() {
res.SetError(errNotOnline, cmds.ErrClient)
return
}
vis, _, err := req.Option("vis").String()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
timeoutS, _, err := req.Option("timeout").String()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
timeout := DefaultDiagnosticTimeout
if timeoutS != "" {
t, err := time.ParseDuration(timeoutS)
if err != nil {
res.SetError(errors.New("error parsing timeout"), cmds.ErrNormal)
return
}
timeout = t
}
info, err := n.Diagnostics.GetDiagnostic(req.Context(), timeout)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
switch vis {
case visD3:
res.SetOutput(bytes.NewReader(diag.GetGraphJson(info)))
case visDot:
buf := new(bytes.Buffer)
w := diag.DotWriter{W: buf}
err := w.WriteGraph(info)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
res.SetOutput(io.Reader(buf))
case visText:
output, err := stdDiagOutputMarshal(standardDiagOutput(info))
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
res.SetOutput(output)
default:
res.SetError(err, cmds.ErrNormal)
return
}
},
}
func stdDiagOutputMarshal(output *DiagnosticOutput) (io.Reader, error) {
buf := new(bytes.Buffer)
err := printDiagnostics(buf, output)
if err != nil {
return nil, err
}
return buf, nil
}
func standardDiagOutput(info []*diag.DiagInfo) *DiagnosticOutput {
output := make([]DiagnosticPeer, len(info))
for i, peer := range info {
connections := make([]DiagnosticConnection, len(peer.Connections))
for j, conn := range peer.Connections {
connections[j] = DiagnosticConnection{
ID: conn.ID,
NanosecondsLatency: uint64(conn.Latency.Nanoseconds()),
Count: conn.Count,
}
}
output[i] = DiagnosticPeer{
ID: peer.ID,
UptimeSeconds: uint64(peer.LifeSpan.Seconds()),
BandwidthBytesIn: peer.BwIn,
BandwidthBytesOut: peer.BwOut,
Connections: connections,
}
}
return &DiagnosticOutput{output}
}
func printDiagnostics(out io.Writer, info *DiagnosticOutput) error {
diagTmpl := `
{{ range $peer := .Peers }}
ID {{ $peer.ID }} up {{ $peer.UptimeSeconds }} seconds connected to {{ len .Connections }}:{{ range $connection := .Connections }}
ID {{ $connection.ID }} connections: {{ $connection.Count }} latency: {{ $connection.NanosecondsLatency }} ns{{ end }}
{{end}}
`
templ, err := template.New("DiagnosticOutput").Parse(diagTmpl)
if err != nil {
return err
}
err = templ.Execute(out, info)
if err != nil {
return err
}
return nil
}
package commands
import (
"bytes"
"testing"
)
func TestPrintDiagnostics(t *testing.T) {
output := DiagnosticOutput{
Peers: []DiagnosticPeer{
{ID: "QmNrjRuUtBNZAigzLRdZGN1YCNUxdF2WY2HnKyEFJqoTeg",
UptimeSeconds: 14,
Connections: []DiagnosticConnection{
{ID: "QmNrjRuUtBNZAigzLRdZGN1YCNUxdF2WY2HnKyEFJqoTeg",
NanosecondsLatency: 1347899,
},
},
},
{ID: "QmUaUZDp6QWJabBYSKfiNmXLAXD8HNKnWZh9Zoz6Zri9Ti",
UptimeSeconds: 14,
},
},
}
buf := new(bytes.Buffer)
if err := printDiagnostics(buf, &output); err != nil {
t.Fatal(err)
}
t.Log(buf.String())
}
......@@ -23,7 +23,6 @@ import (
bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
bserv "github.com/ipfs/go-ipfs/blockservice"
diag "github.com/ipfs/go-ipfs/diagnostics"
exchange "github.com/ipfs/go-ipfs/exchange"
bitswap "github.com/ipfs/go-ipfs/exchange/bitswap"
bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
......@@ -127,7 +126,6 @@ type IpfsNode struct {
Routing routing.IpfsRouting // the routing system. recommend ipfs-dht
Exchange exchange.Interface // the block exchange + strategy (bitswap)
Namesys namesys.NameSystem // the name system, resolves paths to hashes
Diagnostics *diag.Diagnostics // the diagnostics service
Ping *ping.PingService
Reprovider *rp.Reprovider // the value reprovider system
IpnsRepub *ipnsrp.Republisher
......@@ -317,7 +315,6 @@ func (n *IpfsNode) HandlePeerFound(p pstore.PeerInfo) {
// initialized with the host and _before_ we start listening.
func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost.Host, routingOption RoutingOption) error {
// setup diagnostics service
n.Diagnostics = diag.NewDiagnostics(n.Identity, host)
n.Ping = ping.NewPingService(host)
// setup routing service
......
# ipfs diagnostics
Usage:
```sh
ipfs diag net [--vis=<vis>]
```
## view in d3
Install https://github.com/jbenet/ipfs-diag-net-d3-vis then:
```
> ipfs diag net --vis=d3 | d3view
http://ipfs.benet.ai:8080/ipfs/QmX8PuUyhSet8fppZHuRNxG7vk949z7XDxnsAz3zN77MGx#QmdhRqGea2QEzyKHG9Zhkc12d2994iah1h47tfHJifuzhT
```
// package diagnostics implements a network diagnostics service that
// allows a request to traverse the network and gather information
// on every node connected to it.
package diagnostics
import (
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
context "context"
pb "github.com/ipfs/go-ipfs/diagnostics/pb"
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
ctxio "gx/ipfs/QmTKsRYeY4simJyf37K93juSq75Lo8MVCDJ7owjmf46u8W/go-context/io"
inet "gx/ipfs/QmVHSBsn8LEeay8m5ERebgUVuhzw838PsyTttCmP6GMJkg/go-libp2p-net"
ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io"
proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
host "gx/ipfs/QmcyNeWPsoFGxThGpV8JnJdfUNankKhWCTrbrcFRQda4xR/go-libp2p-host"
peer "gx/ipfs/QmdS9KpbDyPrieswibZhkod1oXqRwZJrUPzxCofAMWpFGq/go-libp2p-peer"
)
var log = logging.Logger("diagnostics")
// ProtocolDiag is the diagnostics protocol.ID
var ProtocolDiag protocol.ID = "/ipfs/diag/net/1.0.0"
var ProtocolDiagOld protocol.ID = "/ipfs/diagnostics"
var ErrAlreadyRunning = errors.New("diagnostic with that ID already running")
const ResponseTimeout = time.Second * 10
const HopTimeoutDecrement = time.Second * 2
// Diagnostics is a net service that manages requesting and responding to diagnostic
// requests
type Diagnostics struct {
host host.Host
self peer.ID
diagLock sync.Mutex
diagMap map[string]time.Time
birth time.Time
}
// NewDiagnostics instantiates a new diagnostics service running on the given network
func NewDiagnostics(self peer.ID, h host.Host) *Diagnostics {
d := &Diagnostics{
host: h,
self: self,
birth: time.Now(),
diagMap: make(map[string]time.Time),
}
h.SetStreamHandler(ProtocolDiag, d.handleNewStream)
h.SetStreamHandler(ProtocolDiagOld, d.handleNewStream)
return d
}
type connDiagInfo struct {
Latency time.Duration
ID string
Count int
}
type DiagInfo struct {
// This nodes ID
ID string
// A list of peers this node currently has open connections to
Connections []connDiagInfo
// A list of keys provided by this node
// (currently not filled)
Keys []string
// How long this node has been running for
// TODO rename Uptime
LifeSpan time.Duration
// Incoming Bandwidth Usage
BwIn uint64
// Outgoing Bandwidth Usage
BwOut uint64
// Information about the version of code this node is running
CodeVersion string
}
// Marshal to json
func (di *DiagInfo) Marshal() []byte {
b, err := json.Marshal(di)
if err != nil {
panic(err)
}
//TODO: also consider compressing this. There will be a lot of these
return b
}
func (d *Diagnostics) getPeers() map[peer.ID]int {
counts := make(map[peer.ID]int)
for _, p := range d.host.Network().Peers() {
counts[p]++
}
return counts
}
func (d *Diagnostics) getDiagInfo() *DiagInfo {
di := new(DiagInfo)
di.CodeVersion = "github.com/ipfs/go-ipfs"
di.ID = d.self.Pretty()
di.LifeSpan = time.Since(d.birth)
di.Keys = nil // Currently no way to query datastore
// di.BwIn, di.BwOut = d.host.BandwidthTotals() //TODO fix this.
for p, n := range d.getPeers() {
d := connDiagInfo{
Latency: d.host.Peerstore().LatencyEWMA(p),
ID: p.Pretty(),
Count: n,
}
di.Connections = append(di.Connections, d)
}
return di
}
func newID() string {
id := make([]byte, 16)
rand.Read(id)
return string(id)
}
// GetDiagnostic runs a diagnostics request across the entire network
func (d *Diagnostics) GetDiagnostic(ctx context.Context, timeout time.Duration) ([]*DiagInfo, error) {
log.Debug("getting diagnostic")
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
diagID := newID()
d.diagLock.Lock()
d.diagMap[diagID] = time.Now()
d.diagLock.Unlock()
log.Debug("begin diagnostic")
peers := d.getPeers()
log.Debugf("Sending diagnostic request to %d peers.", len(peers))
pmes := newMessage(diagID)
pmes.SetTimeoutDuration(timeout - HopTimeoutDecrement) // decrease timeout per hop
dpeers, err := d.getDiagnosticFromPeers(ctx, d.getPeers(), pmes)
if err != nil {
return nil, fmt.Errorf("diagnostic from peers err: %s", err)
}
di := d.getDiagInfo()
out := []*DiagInfo{di}
for dpi := range dpeers {
out = append(out, dpi)
}
return out, nil
}
func decodeDiagJson(data []byte) (*DiagInfo, error) {
di := new(DiagInfo)
err := json.Unmarshal(data, di)
if err != nil {
return nil, err
}
return di, nil
}
func (d *Diagnostics) getDiagnosticFromPeers(ctx context.Context, peers map[peer.ID]int, pmes *pb.Message) (<-chan *DiagInfo, error) {
respdata := make(chan *DiagInfo)
wg := sync.WaitGroup{}
for p := range peers {
wg.Add(1)
log.Debugf("Sending diagnostic request to peer: %s", p)
go func(p peer.ID) {
defer wg.Done()
out, err := d.getDiagnosticFromPeer(ctx, p, pmes)
if err != nil {
log.Debugf("Error getting diagnostic from %s: %s", p, err)
return
}
for d := range out {
select {
case respdata <- d:
case <-ctx.Done():
return
}
}
}(p)
}
go func() {
wg.Wait()
close(respdata)
}()
return respdata, nil
}
func (d *Diagnostics) getDiagnosticFromPeer(ctx context.Context, p peer.ID, pmes *pb.Message) (<-chan *DiagInfo, error) {
s, err := d.host.NewStream(ctx, p, ProtocolDiag, ProtocolDiagOld)
if err != nil {
return nil, err
}
cr := ctxio.NewReader(ctx, s) // ok to use. we defer close stream in this func
cw := ctxio.NewWriter(ctx, s) // ok to use. we defer close stream in this func
r := ggio.NewDelimitedReader(cr, inet.MessageSizeMax)
w := ggio.NewDelimitedWriter(cw)
start := time.Now()
if err := w.WriteMsg(pmes); err != nil {
return nil, err
}
out := make(chan *DiagInfo)
go func() {
defer func() {
close(out)
s.Close()
rtt := time.Since(start)
log.Infof("diagnostic request took: %s", rtt.String())
}()
for {
rpmes := new(pb.Message)
if err := r.ReadMsg(rpmes); err != nil {
log.Debugf("Error reading diagnostic from stream: %s", err)
return
}
if rpmes == nil {
log.Debug("got no response back from diag request")
return
}
di, err := decodeDiagJson(rpmes.GetData())
if err != nil {
log.Debug(err)
return
}
select {
case out <- di:
case <-ctx.Done():
return
}
}
}()
return out, nil
}
func newMessage(diagID string) *pb.Message {
pmes := new(pb.Message)
pmes.DiagID = proto.String(diagID)
return pmes
}
func (d *Diagnostics) HandleMessage(ctx context.Context, s inet.Stream) error {
cr := ctxio.NewReader(ctx, s)
cw := ctxio.NewWriter(ctx, s)
r := ggio.NewDelimitedReader(cr, inet.MessageSizeMax) // maxsize
w := ggio.NewDelimitedWriter(cw)
// deserialize msg
pmes := new(pb.Message)
if err := r.ReadMsg(pmes); err != nil {
log.Debugf("Failed to decode protobuf message: %v", err)
return nil
}
// Print out diagnostic
log.Infof("[peer: %s] Got message from [%s]\n",
d.self.Pretty(), s.Conn().RemotePeer())
// Make sure we havent already handled this request to prevent loops
if err := d.startDiag(pmes.GetDiagID()); err != nil {
return nil
}
resp := newMessage(pmes.GetDiagID())
resp.Data = d.getDiagInfo().Marshal()
if err := w.WriteMsg(resp); err != nil {
log.Debugf("Failed to write protobuf message over stream: %s", err)
return err
}
timeout := pmes.GetTimeoutDuration()
if timeout < HopTimeoutDecrement {
return fmt.Errorf("timeout too short: %s", timeout)
}
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
pmes.SetTimeoutDuration(timeout - HopTimeoutDecrement)
dpeers, err := d.getDiagnosticFromPeers(ctx, d.getPeers(), pmes)
if err != nil {
log.Debugf("diagnostic from peers err: %s", err)
return err
}
for b := range dpeers {
resp := newMessage(pmes.GetDiagID())
resp.Data = b.Marshal()
if err := w.WriteMsg(resp); err != nil {
log.Debugf("Failed to write protobuf message over stream: %s", err)
return err
}
}
return nil
}
func (d *Diagnostics) startDiag(id string) error {
d.diagLock.Lock()
_, found := d.diagMap[id]
if found {
d.diagLock.Unlock()
return ErrAlreadyRunning
}
d.diagMap[id] = time.Now()
d.diagLock.Unlock()
return nil
}
func (d *Diagnostics) handleNewStream(s inet.Stream) {
d.HandleMessage(context.Background(), s)
s.Close()
}
include mk/header.mk
PB_$(d) = $(wildcard $(d)/*.proto)
TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
#DEPS_GO += $(TGTS_$(d))
include mk/footer.mk
// Code generated by protoc-gen-gogo.
// source: diagnostics.proto
// DO NOT EDIT!
/*
Package diagnostics_pb is a generated protocol buffer package.
It is generated from these files:
diagnostics.proto
It has these top-level messages:
Message
*/
package diagnostics_pb
import proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = math.Inf
type Message struct {
DiagID *string `protobuf:"bytes,1,req" json:"DiagID,omitempty"`
Data []byte `protobuf:"bytes,2,opt" json:"Data,omitempty"`
Timeout *int64 `protobuf:"varint,3,opt" json:"Timeout,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Message) Reset() { *m = Message{} }
func (m *Message) String() string { return proto.CompactTextString(m) }
func (*Message) ProtoMessage() {}
func (m *Message) GetDiagID() string {
if m != nil && m.DiagID != nil {
return *m.DiagID
}
return ""
}
func (m *Message) GetData() []byte {
if m != nil {
return m.Data
}
return nil
}
func (m *Message) GetTimeout() int64 {
if m != nil && m.Timeout != nil {
return *m.Timeout
}
return 0
}
func init() {
}
package diagnostics.pb;
message Message {
required string DiagID = 1;
optional bytes Data = 2;
optional int64 Timeout = 3; // in nanoseconds
}
package diagnostics_pb
import (
"time"
)
func (m *Message) GetTimeoutDuration() time.Duration {
return time.Duration(m.GetTimeout())
}
func (m *Message) SetTimeoutDuration(t time.Duration) {
it := int64(t)
m.Timeout = &it
}
package diagnostics
import (
"encoding/json"
"fmt"
"io"
rtable "gx/ipfs/QmXKSwZVoHCTne4jTLzDtMc2K6paEZ2QaUMQfJ4ogYd28n/go-libp2p-kbucket"
peer "gx/ipfs/QmdS9KpbDyPrieswibZhkod1oXqRwZJrUPzxCofAMWpFGq/go-libp2p-peer"
)
type node struct {
Name string `json:"name"`
Value uint64 `json:"value"`
RtKey string `json:"rtkey"`
}
type link struct {
Source int `json:"source"`
Target int `json:"target"`
Value int `json:"value"`
}
func GetGraphJson(dinfo []*DiagInfo) []byte {
out := make(map[string]interface{})
names := make(map[string]int)
var nodes []*node
for _, di := range dinfo {
names[di.ID] = len(nodes)
val := di.BwIn + di.BwOut + 10
// include the routing table key, for proper routing table display
rtk := peer.ID(rtable.ConvertPeerID(peer.ID(di.ID))).Pretty()
nodes = append(nodes, &node{Name: di.ID, Value: val, RtKey: rtk})
}
var links []*link
linkexists := make([][]bool, len(nodes))
for i := range linkexists {
linkexists[i] = make([]bool, len(nodes))
}
for _, di := range dinfo {
myid := names[di.ID]
for _, con := range di.Connections {
thisid := names[con.ID]
if !linkexists[thisid][myid] {
links = append(links, &link{
Source: myid,
Target: thisid,
Value: 3,
})
linkexists[myid][thisid] = true
}
}
}
out["nodes"] = nodes
out["links"] = links
b, err := json.Marshal(out)
if err != nil {
panic(err)
}
return b
}
type DotWriter struct {
W io.Writer
err error
}
// Write writes a buffer to the internal writer.
// It handles errors as in: http://blog.golang.org/errors-are-values
func (w *DotWriter) Write(buf []byte) (n int, err error) {
if w.err == nil {
n, w.err = w.W.Write(buf)
}
return n, w.err
}
// WriteS writes a string
func (w *DotWriter) WriteS(s string) (n int, err error) {
return w.Write([]byte(s))
}
func (w *DotWriter) WriteNetHeader(dinfo []*DiagInfo) error {
label := fmt.Sprintf("Nodes: %d\\l", len(dinfo))
w.WriteS("subgraph cluster_L { ")
w.WriteS("L [shape=box fontsize=32 label=\"" + label + "\"] ")
w.WriteS("}\n")
return w.err
}
func (w *DotWriter) WriteNode(i int, di *DiagInfo) error {
box := "[label=\"%s\n%d conns\" fontsize=8 shape=box tooltip=\"%s (%d conns)\"]"
box = fmt.Sprintf(box, di.ID, len(di.Connections), di.ID, len(di.Connections))
w.WriteS(fmt.Sprintf("N%d %s\n", i, box))
return w.err
}
func (w *DotWriter) WriteEdge(i, j int, di *DiagInfo, conn connDiagInfo) error {
n := fmt.Sprintf("%s ... %s (%d)", di.ID, conn.ID, conn.Latency)
s := "[label=\" %d\" weight=%d tooltip=\"%s\" labeltooltip=\"%s\" style=\"dotted\"]"
s = fmt.Sprintf(s, conn.Latency, conn.Count, n, n)
w.WriteS(fmt.Sprintf("N%d -> N%d %s\n", i, j, s))
return w.err
}
func (w *DotWriter) WriteGraph(dinfo []*DiagInfo) error {
w.WriteS("digraph \"diag-net\" {\n")
w.WriteNetHeader(dinfo)
idx := make(map[string]int)
for i, di := range dinfo {
if _, found := idx[di.ID]; found {
log.Debugf("DotWriter skipped duplicate %s", di.ID)
continue
}
idx[di.ID] = i
w.WriteNode(i, di)
}
for i, di := range dinfo {
for _, conn := range di.Connections {
j, found := idx[conn.ID]
if !found { // if we didnt get it earlier...
j = len(idx)
idx[conn.ID] = j
}
w.WriteEdge(i, j, di, conn)
}
}
w.WriteS("}")
return w.err
}
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论