[ May 14 04:13:09 Enabled. ]
[ May 14 04:13:09 Rereading configuration. ]
[ May 14 04:13:10 Rereading configuration. ]
[ May 14 04:13:11 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0cc900c1-53c3-42eb-94ec-d1d543fcb9b9/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0cc900c1-53c3-42eb-94ec-d1d543fcb9b9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:11.665275727Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:11.667262337Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:11.667278278Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:11.69293618Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:11.798871868Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:32.945817213Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61928"}
{"msg":"region 3754c7d6-1b23-4248-86ac-c72c9f559a4c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:32.984527942Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:32.988001352Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"uri":"/crucible/0/regions","method":"POST","req_id":"39cbf356-4906-47a4-b7e8-546287cd98ac","remote_addr":"[fd00:1122:3344:101::a]:61928","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":11717,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:32.988026443Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_0cc900c1-53c3-42eb-94ec-d1d543fcb9b9/crucible/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:32.999989117Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0cc900c1-53c3-42eb-94ec-d1d543fcb9b9/crucible/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.000073629Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3754c7d6-1b23-4248-86ac-c72c9f559a4c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.06149027Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"region":"3754c7d6-1b23-4248-86ac-c72c9f559a4c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3754c7d6-1b23-4248-86ac-c72c9f559a4c\" \"--data\" \"/data/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.061569221Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"region":"3754c7d6-1b23-4248-86ac-c72c9f559a4c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.18468749Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"region":"3754c7d6-1b23-4248-86ac-c72c9f559a4c","component":"worker"}
{"msg":"region 3754c7d6-1b23-4248-86ac-c72c9f559a4c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.184782201Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.185086228Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3754c7d6-1b23-4248-86ac-c72c9f559a4c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.185507176Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3754c7d6-1b23-4248-86ac-c72c9f559a4c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.189386465Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.199525862Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3754c7d6-1b23-4248-86ac-c72c9f559a4c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.20289101Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.203689787Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.205731828Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.205784009Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.20582587Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.217828295Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.222673713Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.361593183Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"uri":"/crucible/0/regions","method":"POST","req_id":"15d31304-c847-4074-a269-409301acaa6c","remote_addr":"[fd00:1122:3344:101::a]:61928","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":115,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.887328123Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37168"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.887998497Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"uri":"/crucible/0/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","method":"GET","req_id":"2520e8d4-0073-4698-a742-df4ecdca1dd3","remote_addr":"[fd00:1122:3344:101::a]:37168","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":653,"response_code":"200"}
{"msg":"path is \"/data/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.88814798Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"datafile"}
{"msg":"dataset is oxp_0cc900c1-53c3-42eb-94ec-d1d543fcb9b9/crucible/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.897554101Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"datafile"}
{"msg":"region 3754c7d6-1b23-4248-86ac-c72c9f559a4c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.906358091Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.906550415Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.908476304Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"uri":"/crucible/0/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","method":"DELETE","req_id":"d8585a35-2ce4-4fa9-a6cd-e77ab1e3cdbe","remote_addr":"[fd00:1122:3344:101::a]:37168","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20357,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.908630697Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"uri":"/crucible/0/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","method":"GET","req_id":"ac15c605-8ca5-449f-8901-8931a6ba284d","remote_addr":"[fd00:1122:3344:101::a]:37168","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-3754c7d6-1b23-4248-86ac-c72c9f559a4c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.909609377Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.913869554Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0cc900c1-53c3-42eb-94ec-d1d543fcb9b9/crucible/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.924401839Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"region":"3754c7d6-1b23-4248-86ac-c72c9f559a4c","component":"worker"}
{"msg":"region 3754c7d6-1b23-4248-86ac-c72c9f559a4c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:33.980770587Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-14T04:13:34.032129293Z","hostname":"oxz_crucible_984ce063-abda-4277-b3e0-28ac8c827e76","pid":22339,"uri":"/crucible/0/regions/3754c7d6-1b23-4248-86ac-c72c9f559a4c","method":"GET","req_id":"7b7fffdd-69fc-45f5-9a70-9e303b03f497","remote_addr":"[fd00:1122:3344:101::a]:37168","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":103,"response_code":"200"}