完成实验二前两个任务；修改gitignore屏蔽.pyc文件

.gitignore

@@ -2,4 +2,6 @@ dataset/
 
 .vscode/
 
-.ipynb_checkpoints/
+.ipynb_checkpoints/
+
+*.pyc

Lab2/code/1.1.py

@@ -0,0 +1,132 @@
+import time
+import numpy as np
+import torch
+from torch.nn.functional import one_hot, softmax
+from torch.utils.data import Dataset, DataLoader
+from torch import nn
+from torchvision import datasets, transforms
+from tqdm import tqdm
+from utils import *
+
+import ipdb
+
+
+class Model_1_1:
+    def __init__(self):
+        self.linear = My_Linear(in_features=500, out_features=1)
+        self.sigmoid = My_Sigmoid()
+        self.params = self.linear.params
+
+    def __call__(self, x):
+        return self.forward(x)
+
+    def forward(self, x):
+        x = self.linear(x)
+        x = self.sigmoid(x)
+        return x
+
+    def to(self, device: str):
+        for param in self.params:
+            param.data = param.data.to(device=device)
+        return self
+
+    def parameters(self):
+        return self.params
+    
+    def train(self):
+        for param in self.params:
+            param.requires_grad = True
+    
+    def eval(self):
+        for param in self.params:
+            param.requires_grad = False
+
+class My_Regression_Dataset(Dataset):
+    def __init__(self, train=True):
+        data_size = 7000 if train else 3000
+        np.random.seed(0)
+        x = np.random.random((data_size, 500)) * 0.005
+        noise = np.random.randn(data_size) * 1e-7
+        y = 0.028 - 0.0056 * x.sum(axis=1) + noise
+        y = y.reshape(-1, 1)
+        self.data = [[x[i], y[i]] for i in range(x.shape[0])]
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, index):
+        x, y = self.data[index]
+        x = torch.FloatTensor(x)
+        y = torch.FloatTensor(y)
+        return x, y
+
+if __name__ == "__main__":
+    learning_rate = 5
+    num_epochs = 10
+    batch_size = 512
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+    train_regression_dataset = My_Regression_Dataset(train=True)
+    test_regression_dataset = My_Regression_Dataset(train=False)
+    train_dataloader = DataLoader(
+        dataset=train_regression_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+    test_dataloader = DataLoader(
+        dataset=test_regression_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+
+    model = Model_1_1().to(device)
+    criterion = My_BCELoss()
+    optimizer = My_optimizer(model.parameters(), lr=learning_rate)
+
+    for epoch in range(num_epochs):
+        model.train()
+        total_epoch_loss = 0
+        start_time = time.time()
+        for index, (x, targets) in tqdm(enumerate(train_dataloader), total=len(train_dataloader)):
+            optimizer.zero_grad()
+            
+            x = x.to(device)
+            targets = targets.to(device)
+            
+            y_pred = model(x)
+            
+            loss = criterion(y_pred, targets)
+            total_epoch_loss += loss.item()
+
+            loss.backward()
+            optimizer.step()
+
+        end_time = time.time()
+        train_time = end_time - start_time
+
+        model.eval()
+        with torch.no_grad():
+            total_epoch_acc = 0
+            start_time = time.time()
+            for index, (x, targets) in tqdm(enumerate(test_dataloader), total=len(test_dataloader)):
+                x = x.to(device)
+                targets = targets.to(device)
+                
+                y_pred = model(x)
+                total_epoch_acc += (1 - torch.abs(y_pred - targets) / torch.abs(targets)).sum().item()
+                
+            end_time = time.time()
+            test_time = end_time - start_time
+            
+        avg_epoch_acc = total_epoch_acc / len(test_regression_dataset)
+        print(
+            f"Epoch [{epoch + 1}/{num_epochs}],",
+            f"Train Loss: {total_epoch_loss},",
+            f"Used Time: {train_time * 1000:.3f}ms,",
+            f"Test Acc: {avg_epoch_acc * 100:.3f}%,",
+            f"Used Time: {test_time * 1000:.3f}ms",
+        )

Lab2/code/1.2.py

@@ -0,0 +1,137 @@
+import time
+import numpy as np
+import torch
+from torch.nn.functional import one_hot, softmax
+from torch.utils.data import Dataset, DataLoader
+from torch import nn
+from torchvision import datasets, transforms
+from tqdm import tqdm
+from utils import *
+
+import ipdb
+
+
+class Model_1_2:
+    def __init__(self):
+        self.fc = My_Linear(in_features=200, out_features=1)
+        self.sigmoid = My_Sigmoid()
+        self.params = self.fc.parameters()
+
+    def __call__(self, x):
+        return self.forward(x)
+
+    def forward(self, x):
+        x = self.fc(x)
+        x = self.sigmoid(x)
+        return x
+
+    def to(self, device: str):
+        for param in self.params:
+            param.data = param.data.to(device=device)
+        return self
+
+    def parameters(self):
+        return self.params
+    
+    def train(self):
+        for param in self.params:
+            param.requires_grad = True
+    
+    def eval(self):
+        for param in self.params:
+            param.requires_grad = False
+
+class My_BinaryCLS_Dataset(Dataset):
+    def __init__(self, train=True, num_features=200):
+        num_samples = 7000 if train else 3000
+        
+        x_1 = np.random.normal(loc=-0.5, scale=0.2, size=(num_samples, num_features))
+        x_2 = np.random.normal(loc=0.5, scale=0.2, size=(num_samples, num_features))
+        
+        labels_1 = np.zeros((num_samples, 1))
+        labels_2 = np.ones((num_samples, 1))
+        
+        x = np.concatenate((x_1, x_2), axis=0)
+        labels = np.concatenate((labels_1, labels_2), axis=0)
+        self.data = [[x[i], labels[i]] for i in range(2 * num_samples)]
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, index):
+        x, y = self.data[index]
+        x = torch.FloatTensor(x)
+        y = torch.LongTensor(y)
+        return x, y
+    
+
+if __name__ == "__main__":
+    learning_rate = 5e-3
+    num_epochs = 10
+    batch_size = 512
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+    train_binarycls_dataset = My_BinaryCLS_Dataset(train=True)
+    test_binarycls_dataset = My_BinaryCLS_Dataset(train=False)
+    train_dataloader = DataLoader(
+        dataset=train_binarycls_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+    test_dataloader = DataLoader(
+        dataset=test_binarycls_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+
+    model = Model_1_2().to(device)
+    criterion = My_BCELoss()
+    optimizer = My_optimizer(model.parameters(), lr=learning_rate)
+
+    for epoch in range(num_epochs):
+        model.train()
+        total_epoch_loss = 0
+        start_time = time.time()
+        for index, (x, targets) in tqdm(enumerate(train_dataloader), total=len(train_dataloader)):
+            optimizer.zero_grad()
+
+            x = x.to(device)
+            targets = targets.to(device).to(dtype=torch.float)
+            
+            y_pred = model(x)
+            loss = criterion(y_pred, targets)
+            total_epoch_loss += loss.item()
+
+            loss.backward()
+            optimizer.step()
+
+        end_time = time.time()
+        train_time = end_time - start_time
+
+        model.eval()
+        with torch.no_grad():
+            total_epoch_acc = 0
+            start_time = time.time()
+            for index, (x, targets) in tqdm(enumerate(test_dataloader), total=len(test_dataloader)):
+                x = x.to(device)
+                targets = targets.to(device)
+                
+                output = model(x)
+                pred = (output > 0.5).to(dtype=torch.long)
+                total_epoch_acc += (pred == targets).sum().item()
+                
+            end_time = time.time()
+            test_time = end_time - start_time
+
+        avg_epoch_acc = total_epoch_acc / len(test_binarycls_dataset)
+        print(
+            f"Epoch [{epoch + 1}/{num_epochs}],",
+            f"Train Loss: {total_epoch_loss},",
+            f"Used Time: {train_time * 1000:.3f}ms,",
+            f"Test Acc: {avg_epoch_acc * 100:.3f}%,",
+            f"Used Time: {test_time * 1000:.3f}ms",
+        )

Lab2/code/1.3.py

@@ -0,0 +1,125 @@
+import time
+import numpy as np
+import torch
+from torch.nn.functional import one_hot, softmax
+from torch.utils.data import Dataset, DataLoader
+from torch import nn
+from torchvision import datasets, transforms
+from tqdm import tqdm
+from utils import *
+
+import ipdb
+        
+
+class Model_1_3:
+    def __init__(self, num_classes):
+        self.flatten = My_Flatten()
+        self.linear = My_Linear(in_features=28 * 28, out_features=num_classes)
+        self.params = self.linear.params
+
+    def __call__(self, x: torch.Tensor):
+        return self.forward(x)
+
+    def forward(self, x: torch.Tensor):
+        x = self.flatten(x)
+        x = self.linear(x)
+        return x
+
+    def to(self, device: str):
+        for param in self.params:
+            param.data = param.data.to(device=device)
+        return self
+
+    def parameters(self):
+        return self.params
+    
+    def train(self):
+        for param in self.params:
+            param.requires_grad = True
+    
+    def eval(self):
+        for param in self.params:
+            param.requires_grad = False
+
+if __name__ == "__main__":
+    learning_rate = 1e-1
+    num_epochs = 10
+    batch_size = 512
+    num_classes = 10
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+    transform = transforms.Compose(
+        [
+            transforms.ToTensor(),
+            transforms.Normalize((0.5,), (0.5,)),
+        ]
+    )
+    train_mnist_dataset = datasets.MNIST(root="../dataset", train=True, transform=transform, download=True)
+    test_mnist_dataset = datasets.MNIST(root="../dataset", train=False, transform=transform, download=True)
+    train_loader = DataLoader(
+        dataset=train_mnist_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+    test_loader = DataLoader(
+        dataset=test_mnist_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+
+    model = Model_1_3(num_classes).to(device)
+    criterion = My_CrossEntropyLoss()
+    optimizer = My_optimizer(model.parameters(), lr=learning_rate)
+    
+    for epoch in range(num_epochs):
+        model.train()
+        total_epoch_loss = 0
+        start_time = time.time()
+        for index, (images, targets) in tqdm(
+            enumerate(train_loader), total=len(train_loader)
+        ):
+            optimizer.zero_grad()
+
+            images = images.to(device)
+            targets = targets.to(device)
+            one_hot_targets = my_one_hot(targets, num_classes=num_classes).to(dtype=torch.float)
+
+            outputs = model(images)
+            loss = criterion(outputs, one_hot_targets)
+            total_epoch_loss += loss.item()
+
+            loss.backward()
+            optimizer.step()
+
+        end_time = time.time()
+        train_time = end_time - start_time
+
+        model.eval()
+        with torch.no_grad():
+            total_epoch_acc = 0
+            start_time = time.time()
+            for index, (image, targets) in tqdm(
+                enumerate(test_loader), total=len(test_loader)
+            ):
+                image = image.to(device)
+                targets = targets.to(device)
+                
+                outputs = model(image)
+                pred = my_softmax(outputs, dim=1)
+                total_epoch_acc += (pred.argmax(1) == targets).sum().item()
+            
+            end_time = time.time()
+            test_time = end_time - start_time
+        
+        avg_epoch_acc = total_epoch_acc / len(test_mnist_dataset)
+        print(
+            f"Epoch [{epoch + 1}/{num_epochs}],",
+            f"Train Loss: {total_epoch_loss},",
+            f"Used Time: {train_time * 1000:.3f}ms,",
+            f"Test Acc: {avg_epoch_acc * 100:.3f}%,",
+            f"Used Time: {test_time * 1000:.3f}ms",
+        )

Lab2/code/2.1.py

@@ -0,0 +1,114 @@
+import time
+import numpy as np
+import torch
+from torch.nn.functional import one_hot, softmax
+from torch.utils.data import Dataset, DataLoader
+from torch import nn
+from torchvision import datasets, transforms
+from tqdm import tqdm
+from utils import *
+
+import ipdb
+
+
+class Model_2_1(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.linear = nn.Linear(in_features=500, out_features=1)
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, x):
+        x = self.linear(x)
+        x = self.sigmoid(x)
+        return x
+    
+
+class My_Regression_Dataset(Dataset):
+    def __init__(self, train=True):
+        data_size = 7000 if train else 3000
+        np.random.seed(0)
+        x = np.random.random((data_size, 500)) * 0.005
+        noise = np.random.randn(data_size) * 1e-7
+        y = 0.028 - 0.0056 * x.sum(axis=1) + noise
+        y = y.reshape(-1, 1)
+        self.data = [[x[i], y[i]] for i in range(x.shape[0])]
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, index):
+        x, y = self.data[index]
+        x = torch.FloatTensor(x)
+        y = torch.FloatTensor(y)
+        return x, y
+
+if __name__ == "__main__":
+    learning_rate = 5
+    num_epochs = 10
+    batch_size = 512
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+    train_regression_dataset = My_Regression_Dataset(train=True)
+    test_regression_dataset = My_Regression_Dataset(train=False)
+    train_dataloader = DataLoader(
+        dataset=train_regression_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+    test_dataloader = DataLoader(
+        dataset=test_regression_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+
+    model = Model_2_1().to(device)
+    criterion = nn.BCELoss()
+    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
+
+    for epoch in range(num_epochs):
+        model.train()
+        total_epoch_loss = 0
+        start_time = time.time()
+        for index, (x, targets) in tqdm(enumerate(train_dataloader), total=len(train_dataloader)):
+            optimizer.zero_grad()
+            
+            x = x.to(device)
+            targets = targets.to(device)
+            
+            y_pred = model(x)
+            
+            loss = criterion(y_pred, targets)
+            total_epoch_loss += loss.item()
+
+            loss.backward()
+            optimizer.step()
+
+        end_time = time.time()
+        train_time = end_time - start_time
+
+        model.eval()
+        with torch.no_grad():
+            total_epoch_acc = 0
+            start_time = time.time()
+            for index, (x, targets) in tqdm(enumerate(test_dataloader), total=len(test_dataloader)):
+                x = x.to(device)
+                targets = targets.to(device)
+                
+                y_pred = model(x)
+                total_epoch_acc += (1 - torch.abs(y_pred - targets) / torch.abs(targets)).sum().item()
+                
+            end_time = time.time()
+            test_time = end_time - start_time
+            
+        avg_epoch_acc = total_epoch_acc / len(test_regression_dataset)
+        print(
+            f"Epoch [{epoch + 1}/{num_epochs}],",
+            f"Train Loss: {total_epoch_loss},",
+            f"Used Time: {train_time * 1000:.3f}ms,",
+            f"Test Acc: {avg_epoch_acc * 100:.3f}%,",
+            f"Used Time: {test_time * 1000:.3f}ms",
+        )

Lab2/code/2.2.py

@@ -0,0 +1,118 @@
+import time
+import numpy as np
+import torch
+from torch.nn.functional import one_hot, softmax
+from torch.utils.data import Dataset, DataLoader
+from torch import nn
+from torchvision import datasets, transforms
+from tqdm import tqdm
+from utils import *
+
+import ipdb
+
+
+class Model_2_1(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.fc = nn.Linear(in_features=200, out_features=1)
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, x):
+        x = self.fc(x)
+        x = self.sigmoid(x)
+        return x
+
+class My_BinaryCLS_Dataset(Dataset):
+    def __init__(self, train=True, num_features=200):
+        num_samples = 7000 if train else 3000
+        
+        x_1 = np.random.normal(loc=-0.5, scale=0.2, size=(num_samples, num_features))
+        x_2 = np.random.normal(loc=0.5, scale=0.2, size=(num_samples, num_features))
+        
+        labels_1 = np.zeros((num_samples, 1))
+        labels_2 = np.ones((num_samples, 1))
+        
+        x = np.concatenate((x_1, x_2), axis=0)
+        labels = np.concatenate((labels_1, labels_2), axis=0)
+        self.data = [[x[i], labels[i]] for i in range(2 * num_samples)]
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, index):
+        x, y = self.data[index]
+        x = torch.FloatTensor(x)
+        y = torch.LongTensor(y)
+        return x, y
+    
+
+if __name__ == "__main__":
+    learning_rate = 1e-4
+    num_epochs = 10
+    batch_size = 512
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+    train_binarycls_dataset = My_BinaryCLS_Dataset(train=True)
+    test_binarycls_dataset = My_BinaryCLS_Dataset(train=False)
+    train_dataloader = DataLoader(
+        dataset=train_binarycls_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+    test_dataloader = DataLoader(
+        dataset=test_binarycls_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+
+    model = Model_2_1().to(device)
+    criterion = nn.BCELoss()
+    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
+
+    for epoch in range(num_epochs):
+        model.train()
+        total_epoch_loss = 0
+        start_time = time.time()
+        for index, (x, targets) in tqdm(enumerate(train_dataloader), total=len(train_dataloader)):
+            optimizer.zero_grad()
+
+            x = x.to(device)
+            targets = targets.to(device).to(dtype=torch.float32)
+            
+            y_pred = model(x)
+            loss = criterion(y_pred, targets)
+            total_epoch_loss += loss.item()
+
+            loss.backward()
+            optimizer.step()
+
+        end_time = time.time()
+        train_time = end_time - start_time
+
+        model.eval()
+        with torch.no_grad():
+            total_epoch_acc = 0
+            start_time = time.time()
+            for index, (x, targets) in tqdm(enumerate(test_dataloader), total=len(test_dataloader)):
+                x = x.to(device)
+                targets = targets.to(device)
+                
+                output = model(x)
+                pred = (output > 0.5).to(dtype=torch.long)
+                total_epoch_acc += (pred == targets).sum().item()
+                
+            end_time = time.time()
+            test_time = end_time - start_time
+
+        avg_epoch_acc = total_epoch_acc / len(test_binarycls_dataset)
+        print(
+            f"Epoch [{epoch + 1}/{num_epochs}],",
+            f"Train Loss: {total_epoch_loss},",
+            f"Used Time: {train_time * 1000:.3f}ms,",
+            f"Test Acc: {avg_epoch_acc * 100:.3f}%,",
+            f"Used Time: {test_time * 1000:.3f}ms",
+        )

Lab2/code/2.3.py

@@ -0,0 +1,106 @@
+import time
+import numpy as np
+import torch
+from torch.nn.functional import one_hot, softmax
+from torch.utils.data import Dataset, DataLoader
+from torch import nn
+from torchvision import datasets, transforms
+from tqdm import tqdm
+from utils import *
+
+import ipdb
+        
+
+class Model_2_3(nn.Module):
+    def __init__(self, num_classes):
+        super().__init__()
+        self.flatten = nn.Flatten()
+        self.linear = nn.Linear(in_features=28 * 28, out_features=num_classes)
+
+    def forward(self, x: torch.Tensor):
+        x = self.flatten(x)
+        x = self.linear(x)
+        return x
+
+if __name__ == "__main__":
+    learning_rate = 5e-2
+    num_epochs = 10
+    batch_size = 512
+    num_classes = 10
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+    transform = transforms.Compose(
+        [
+            transforms.ToTensor(),
+            transforms.Normalize((0.5,), (0.5,)),
+        ]
+    )
+    train_mnist_dataset = datasets.MNIST(root="../dataset", train=True, transform=transform, download=True)
+    test_mnist_dataset = datasets.MNIST(root="../dataset", train=False, transform=transform, download=True)
+    train_loader = DataLoader(
+        dataset=train_mnist_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+    test_loader = DataLoader(
+        dataset=test_mnist_dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=14,
+        pin_memory=True,
+    )
+
+    model = Model_2_3(num_classes).to(device)
+    criterion = nn.CrossEntropyLoss()
+    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
+    
+    for epoch in range(num_epochs):
+        model.train()
+        total_epoch_loss = 0
+        start_time = time.time()
+        for index, (images, targets) in tqdm(
+            enumerate(train_loader), total=len(train_loader)
+        ):
+            optimizer.zero_grad()
+
+            images = images.to(device)
+            targets = targets.to(device)
+            one_hot_targets = one_hot(targets, num_classes=num_classes).to(dtype=torch.float)
+
+            outputs = model(images)
+            loss = criterion(outputs, one_hot_targets)
+            total_epoch_loss += loss.item()
+
+            loss.backward()
+            optimizer.step()
+
+        end_time = time.time()
+        train_time = end_time - start_time
+
+        model.eval()
+        with torch.no_grad():
+            total_epoch_acc = 0
+            start_time = time.time()
+            for index, (image, targets) in tqdm(
+                enumerate(test_loader), total=len(test_loader)
+            ):
+                image = image.to(device)
+                targets = targets.to(device)
+                
+                outputs = model(image)
+                pred = softmax(outputs, dim=1)
+                total_epoch_acc += (pred.argmax(1) == targets).sum().item()
+            
+            end_time = time.time()
+            test_time = end_time - start_time
+        
+        avg_epoch_acc = total_epoch_acc / len(test_mnist_dataset)
+        print(
+            f"Epoch [{epoch + 1}/{num_epochs}],",
+            f"Train Loss: {total_epoch_loss},",
+            f"Used Time: {train_time * 1000:.3f}ms,",
+            f"Test Acc: {avg_epoch_acc * 100:.3f}%,",
+            f"Used Time: {test_time * 1000:.3f}ms",
+        )

Lab2/code/utils.py

@@ -0,0 +1,105 @@
+import time
+import numpy as np
+import torch
+from torch.nn.functional import one_hot, softmax
+from torch.utils.data import Dataset, DataLoader
+from torch import nn
+from torchvision import datasets, transforms
+from tqdm import tqdm
+
+import ipdb
+
+
+# 手动实现torch.nn.functional.one_hot
+def my_one_hot(indices: torch.Tensor, num_classes: int):
+    one_hot_tensor = torch.zeros(len(indices), num_classes, dtype=torch.long).to(indices.device)
+    one_hot_tensor.scatter_(1, indices.view(-1, 1), 1)
+    return one_hot_tensor
+
+
+# 手动实现torch.nn.functional.softmax
+def my_softmax(predictions: torch.Tensor, dim: int):
+    max_values = torch.max(predictions, dim=dim, keepdim=True).values
+    exp_values = torch.exp(predictions - max_values)
+    softmax_output = exp_values / torch.sum(exp_values, dim=dim, keepdim=True)
+    return softmax_output
+
+# 手动实现torch.nn.Linear
+class My_Linear:
+    def __init__(self, in_features: int, out_features: int):
+        self.weight = torch.normal(mean=0.001, std=0.5, size=(out_features, in_features), requires_grad=True, dtype=torch.float32)
+        self.bias = torch.normal(mean=0.001, std=0.5, size=(1,), requires_grad=True, dtype=torch.float32)
+        self.params = [self.weight, self.bias]
+
+    def __call__(self, x):
+        return self.forward(x)
+
+    def forward(self, x):
+        x = torch.matmul(x, self.weight.T) + self.bias
+        return x
+
+    def to(self, device: str):
+        for param in self.params:
+            param.data = param.data.to(device=device)
+        return self
+
+    def parameters(self):
+        return self.params
+
+        
+# 手动实现torch.nn.Flatten
+class My_Flatten:
+    def __call__(self, x: torch.Tensor):
+        x = x.view(x.shape[0], -1)
+        return x
+
+        
+# 手动实现torch.nn.ReLU
+class My_ReLU():
+    def __call__(self, x: torch.Tensor):
+        x = torch.max(x, torch.tensor(0.0, device=x.device))
+        return x
+
+
+# 手动实现torch.nn.Sigmoid
+class My_Sigmoid():
+    def __call__(self, x: torch.Tensor):
+        x = 1. / (1. + torch.exp(-x))
+        return x
+
+
+# 手动实现torch.nn.BCELoss
+class My_BCELoss:
+    def __call__(self, prediction: torch.Tensor, target: torch.Tensor):
+        loss = -torch.mean(target * torch.log(prediction) + (1 - target) * torch.log(1 - prediction))
+        return loss
+
+
+# 手动实现torch.nn.CrossEntropyLoss
+class My_CrossEntropyLoss:
+    def __call__(self, predictions: torch.Tensor, targets: torch.Tensor):
+        max_values = torch.max(predictions, dim=1, keepdim=True).values
+        exp_values = torch.exp(predictions - max_values)
+        softmax_output = exp_values / torch.sum(exp_values, dim=1, keepdim=True)
+
+        log_probs = torch.log(softmax_output)
+        nll_loss = -torch.sum(targets * log_probs, dim=1)
+        average_loss = torch.mean(nll_loss)
+        return average_loss
+
+
+# 手动实现torch.optim.SGD
+class My_optimizer:
+    def __init__(self, params: list[torch.Tensor], lr: float):
+        self.params = params
+        self.lr = lr
+
+    def step(self):
+        with torch.no_grad():
+            for param in self.params:
+                param.data = param.data - self.lr * param.grad.data
+
+    def zero_grad(self):
+        for param in self.params:
+            if param.grad is not None:
+                param.grad.data = torch.zeros_like(param.grad.data)