[ Oct 31 20:20:48 Enabled. ]
[ Oct 31 20:20:48 Rereading configuration. ]
[ Oct 31 20:20:50 Rereading configuration. ]
[ Oct 31 20:20:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_29756246-aa82-409a-9346-5743a657eff3/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_29756246-aa82-409a-9346-5743a657eff3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:20:53.609930546Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:20:53.615673497Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:20:53.615695668Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:20:53.652840809Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:20:53.909411312Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.382373336Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60640"}
{"msg":"region bf811f60-1e95-4804-93d8-4928ab8690fe state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.411318973Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.415417585Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"uri":"/crucible/0/regions","method":"POST","req_id":"126d3c5b-dc33-491a-8c92-b88bdc07015a","remote_addr":"[fd00:1122:3344:101::c]:60640","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":15481,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.416100367Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_29756246-aa82-409a-9346-5743a657eff3/crucible/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.425262828Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_29756246-aa82-409a-9346-5743a657eff3/crucible/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.425302529Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bf811f60-1e95-4804-93d8-4928ab8690fe\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bf811f60-1e95-4804-93d8-4928ab8690fe\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.540591592Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"region":"bf811f60-1e95-4804-93d8-4928ab8690fe","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bf811f60-1e95-4804-93d8-4928ab8690fe\" \"--data\" \"/data/regions/bf811f60-1e95-4804-93d8-4928ab8690fe\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.544554741Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"region":"bf811f60-1e95-4804-93d8-4928ab8690fe","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.647793683Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"uri":"/crucible/0/regions","method":"POST","req_id":"c5ccdea2-d89f-4fc7-a0d3-5282f1a1def2","remote_addr":"[fd00:1122:3344:101::c]:60640","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.762929683Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"region":"bf811f60-1e95-4804-93d8-4928ab8690fe","component":"worker"}
{"msg":"region bf811f60-1e95-4804-93d8-4928ab8690fe state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.762980044Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.763060365Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bf811f60-1e95-4804-93d8-4928ab8690fe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.763441272Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bf811f60-1e95-4804-93d8-4928ab8690fe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.767042455Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.785975367Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bf811f60-1e95-4804-93d8-4928ab8690fe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.789611361Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.790352064Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.792320589Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.79238202Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.792424321Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.80723378Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:27.814889735Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:28.135829017Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"uri":"/crucible/0/regions","method":"POST","req_id":"b02ab00f-2a3d-4820-8a96-6d88f891a4ac","remote_addr":"[fd00:1122:3344:101::c]:60640","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.76310342Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"uri":"/crucible/0/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","method":"GET","req_id":"ff85344d-be0a-4c26-808d-3b41010a313b","remote_addr":"[fd00:1122:3344:101::c]:60640","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":121,"response_code":"200"}
{"msg":"path is \"/data/regions/bf811f60-1e95-4804-93d8-4928ab8690fe\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.771028899Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"datafile"}
{"msg":"dataset is oxp_29756246-aa82-409a-9346-5743a657eff3/crucible/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.7813787Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"datafile"}
{"msg":"region bf811f60-1e95-4804-93d8-4928ab8690fe state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.790411489Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.790545641Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-bf811f60-1e95-4804-93d8-4928ab8690fe (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.79158759Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.793045365Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"uri":"/crucible/0/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","method":"DELETE","req_id":"43570beb-02b3-41a2-b107-bf9deeb9f63b","remote_addr":"[fd00:1122:3344:101::c]:60640","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22060,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.796847952Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"uri":"/crucible/0/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","method":"GET","req_id":"d8033bd4-49ca-4b9e-b147-509aa04727c8","remote_addr":"[fd00:1122:3344:101::c]:60640","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.797085446Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_29756246-aa82-409a-9346-5743a657eff3/crucible/regions/bf811f60-1e95-4804-93d8-4928ab8690fe\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.809533403Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"region":"bf811f60-1e95-4804-93d8-4928ab8690fe","component":"worker"}
{"msg":"region bf811f60-1e95-4804-93d8-4928ab8690fe state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.888669593Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-31T20:21:29.903707887Z","hostname":"oxz_crucible_092c2686-49d9-4e5c-a2dc-c32f613d6678","pid":22534,"uri":"/crucible/0/regions/bf811f60-1e95-4804-93d8-4928ab8690fe","method":"GET","req_id":"ef19e9f4-7146-4c24-ae3c-eb03abc20af7","remote_addr":"[fd00:1122:3344:101::c]:60640","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":83,"response_code":"200"}