Upload NeuralCode9.py

NeuralCode9.py

@@ -0,0 +1,469 @@
+import math
+import random
+import time
+import os # Added for path joining
+
+# CUSTOMIZATION
+
+CONTEXT_WINDOW = 4
+EPOCHS = 100
+LR = 0.01
+
+def relu(x):
+    return max(0.0, x)
+
+def stable_softmax(x_list):
+    if not x_list:
+        return []
+    m = max(x_list)
+    exps = [math.exp(i - m) for i in x_list]
+    s = sum(exps)
+    if s == 0:
+        return [1.0 / len(x_list)] * len(x_list)
+    return [e / s for e in exps]
+
+class NeuralNetwork:
+    def __init__(self, layer_sizes=None, activation='relu', output_activation='softmax',
+                 init_range=0.1, grad_clip=1.0, seed=None, context_window=5):
+        if seed is not None:
+            random.seed(seed)
+        self.layer_sizes = layer_sizes[:] if layer_sizes is not None else None
+        self.activation = relu if activation == 'relu' else (lambda x: x)
+        self.output_activation = stable_softmax if output_activation == 'softmax' else (lambda x: x)
+        self.init_range = float(init_range)
+        self.grad_clip = grad_clip
+        self.context_window = context_window
+        self.weights = []
+        self.biases = []
+        self.vocab = []
+        self.word_to_idx = {}
+        self.idx_to_word = {}
+
+    def prepare_data_with_context(self, text):
+        words = [w.strip() for w in text.replace('\n', ' ').split(' ') if w.strip()]
+        self.vocab = sorted(list(set(words)))
+        self.word_to_idx = {w: i for i, w in enumerate(self.vocab)}
+        self.idx_to_word = {i: w for w, i in self.word_to_idx.items()}
+        
+        vocab_size = len(self.vocab)
+        X = []
+        Y = []
+        
+        for i in range(len(words) - self.context_window):
+            context_words = words[i : i + self.context_window]
+            target_word = words[i + self.context_window]
+            
+            x = [0.0] * vocab_size
+            for word in context_words:
+                if word in self.word_to_idx:
+                    x[self.word_to_idx[word]] = 1.0 
+            
+            y = [0.0] * vocab_size
+            if target_word in self.word_to_idx:
+                y[self.word_to_idx[target_word]] = 1.0
+            
+            X.append(x)
+            Y.append(y)
+            
+        return X, Y
+
+    def initialize_weights(self):
+        if self.layer_sizes is None:
+            raise ValueError("layer_sizes must be set before initializing weights.")
+        if self.weights:
+            return
+        for i in range(len(self.layer_sizes) - 1):
+            in_dim = self.layer_sizes[i]
+            out_dim = self.layer_sizes[i + 1]
+            W = [[random.uniform(-self.init_range, self.init_range) for _ in range(out_dim)] for _ in range(in_dim)]
+            b = [0.0 for _ in range(out_dim)]
+            self.weights.append(W)
+            self.biases.append(b)
+
+    def forward(self, x):
+        a = x[:]
+        for i in range(len(self.weights) - 1):
+            next_a = []
+            W = self.weights[i]
+            b = self.biases[i]
+            out_dim = len(W[0])
+            for j in range(out_dim):
+                s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
+                next_a.append(self.activation(s))
+            a = next_a
+
+        W = self.weights[-1]
+        b = self.biases[-1]
+        out = []
+        out_dim = len(W[0])
+        for j in range(out_dim):
+            s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
+            out.append(s)
+        return self.output_activation(out)
+
+    def train(self, training_data, lr=0.01, epochs=500, verbose_every=50):
+        X, Y = self.prepare_data_with_context(training_data)
+        if not X:
+            raise ValueError("Not enough tokens in training data to create context windows.")
+
+        vocab_size = len(self.vocab)
+        if self.layer_sizes is None:
+            self.layer_sizes = [vocab_size, 64, vocab_size]
+        else:
+            self.layer_sizes[0] = vocab_size
+            self.layer_sizes[-1] = vocab_size
+
+        self.initialize_weights()
+
+        for epoch in range(epochs):
+            total_loss = 0.0
+            indices = list(range(len(X)))
+            random.shuffle(indices)
+
+            for idx in indices:
+                x = X[idx]
+                y = Y[idx]
+
+                activations = [x[:]]
+                pre_acts = []
+                a = x[:]
+                
+                for i in range(len(self.weights) - 1):
+                    W, b = self.weights[i], self.biases[i]
+                    z = []
+                    out_dim = len(W[0])
+                    for j in range(out_dim):
+                        s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
+                        z.append(s)
+                    pre_acts.append(z)
+                    a = [self.activation(val) for val in z]
+                    activations.append(a)
+
+                W, b = self.weights[-1], self.biases[-1]
+                z_final = []
+                out_dim = len(W[0])
+                for j in range(out_dim):
+                    s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
+                    z_final.append(s)
+                pre_acts.append(z_final)
+                out = self.output_activation(z_final)
+
+                delta = [out[j] - y[j] for j in range(len(y))]
+
+                for i in reversed(range(len(self.weights))):
+                    in_act = activations[i]
+                    in_dim = len(in_act)
+                    out_dim = len(delta)
+                    
+                    db = delta[:]
+                    if self.grad_clip is not None:
+                        db = [max(-self.grad_clip, min(self.grad_clip, g)) for g in db]
+                    for j in range(len(self.biases[i])):
+                        self.biases[i][j] -= lr * db[j]
+
+                    for k in range(in_dim):
+                        for j in range(out_dim):
+                            grad_w = in_act[k] * delta[j]
+                            if self.grad_clip is not None:
+                                grad_w = max(-self.grad_clip, min(self.grad_clip, grad_w))
+                            self.weights[i][k][j] -= lr * grad_w
+
+                    if i != 0:
+                        prev_delta = [0.0] * in_dim
+                        for p in range(in_dim):
+                            s = sum(self.weights[i][p][j] * delta[j] for j in range(out_dim))
+                            if pre_acts[i-1][p] > 0:
+                                prev_delta[p] = s
+                        delta = prev_delta
+            
+            if epoch % verbose_every == 0 or epoch == epochs - 1:
+                loss = 0.0
+                for x_val, y_val in zip(X, Y):
+                    p = self.forward(x_val)
+                    for j in range(len(y_val)):
+                        if y_val[j] > 0:
+                            loss -= math.log(p[j] + 1e-12)
+                print(f"Epoch {epoch}, Loss: {loss / len(X):.6f}")
+
+    def export_to_python(self, filename):
+        lines = []
+        lines.append("import math\n")
+        lines.append("import time\n\n")
+        lines.append("def relu(x):\n    return max(0.0, x)\n\n")
+        lines.append("def softmax(x_list):\n")
+        lines.append("    if not x_list:\n")
+        lines.append("        return []\n")
+        lines.append("    m = max(x_list)\n")
+        lines.append("    exps = [math.exp(i - m) for i in x_list]\n")
+        lines.append("    s = sum(exps)\n")
+        lines.append("    if s == 0:\n")
+        lines.append("        return [1.0 / len(x_list)] * len(x_list)\n")
+        lines.append("    return [e / s for e in exps]\n\n")
+
+        neuron_id = 0
+        for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)):
+            in_dim, out_dim = len(W), len(W[0])
+            for j in range(out_dim):
+                terms = " + ".join([f"{W[i][j]:.8f}*inputs[{i}]" for i in range(in_dim)]) or "0.0"
+                b_term = f"{b[j]:.8f}"
+                if layer_idx != len(self.weights) - 1:
+                    lines.append(f"def neuron_{neuron_id}(inputs):\n    return relu({terms} + {b_term})\n\n")
+                else:
+                    lines.append(f"def neuron_{neuron_id}(inputs):\n    return {terms} + {b_term}\n\n")
+                neuron_id += 1
+
+        neuron_counter = 0
+        for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)):
+            out_dim = len(W[0])
+            lines.append(f"def layer_{layer_idx}(inputs):\n")
+            inner = ", ".join([f"neuron_{neuron_counter + j}(inputs)" for j in range(out_dim)])
+            lines.append(f"    return [{inner}]\n\n")
+            neuron_counter += out_dim
+
+        lines.append("def predict(inputs):\n")
+        lines.append("    a = inputs\n")
+        for i in range(len(self.weights)):
+            lines.append(f"    a = layer_{i}(a)\n")
+        lines.append("    return softmax(a)\n\n")
+
+        lines.append(f"vocab = {self.vocab}\n")
+        lines.append(f"word_to_idx = {{w: i for i, w in enumerate(vocab)}}\n")
+        lines.append(f"context_window = {self.context_window}\n\n")
+
+        lines.append("if __name__ == '__main__':\n")
+        lines.append("    print('Interactive multi-word text completion.')\n")
+        lines.append("    print(f'Model context window: {context_window} words. Type text or empty to exit.')\n")
+        lines.append("    while True:\n")
+        lines.append("        inp = input('> ').strip()\n")
+        lines.append("        if not inp:\n")
+        lines.append("            break\n")
+        lines.append("        words = [w.strip() for w in inp.split(' ') if w.strip()]\n")
+        lines.append("        generated_words = words[:]\n")
+        lines.append("        print('Input:', ' '.join(generated_words), end='', flush=True)\n")
+        lines.append("        for _ in range(20):\n")
+        lines.append("            context = generated_words[-context_window:]\n")
+        lines.append("            x = [0.0] * len(vocab)\n")
+        lines.append("            for word in context:\n")
+        lines.append("                if word in word_to_idx:\n")
+        lines.append("                    x[word_to_idx[word]] = 1.0\n")
+        lines.append("            out = predict(x)\n")
+        lines.append("            idx = out.index(max(out))\n")
+        lines.append("            next_word = vocab[idx]\n")
+        lines.append("            if next_word == '<|endoftext|>': break\n")
+        lines.append("            generated_words.append(next_word)\n")
+        lines.append("            print(' ' + next_word, end='', flush=True)\n")
+        lines.append("            time.sleep(0.1)\n")
+        lines.append("        print('\\n')\n")
+
+        with open(filename, "w") as f:
+            f.writelines(lines)
+        print(f"Exported network to {filename}")
+
+    def export_to_js(self, base_filename):
+        js_filename = base_filename + ".js"
+        html_filename = base_filename + ".html"
+        
+        # --- Create JavaScript File ---
+        js_lines = []
+        js_lines.append("'use strict';\n\n")
+        js_lines.append("function relu(x) {\n    return Math.max(0.0, x);\n}\n\n")
+        js_lines.append("function softmax(x_list) {\n")
+        js_lines.append("    if (!x_list || x_list.length === 0) return [];\n")
+        js_lines.append("    const m = Math.max(...x_list);\n")
+        js_lines.append("    const exps = x_list.map(x => Math.exp(x - m));\n")
+        js_lines.append("    const s = exps.reduce((a, b) => a + b, 0);\n")
+        js_lines.append("    if (s === 0) return Array(x_list.length).fill(1.0 / x_list.length);\n")
+        js_lines.append("    return exps.map(e => e / s);\n}\n\n")
+
+        neuron_id = 0
+        for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)):
+            in_dim, out_dim = len(W), len(W[0])
+            for j in range(out_dim):
+                terms = " + ".join([f"{W[i][j]:.8f} * inputs[{i}]" for i in range(in_dim)]) or "0.0"
+                b_term = f"{b[j]:.8f}"
+                js_lines.append(f"function neuron_{neuron_id}(inputs) {{\n")
+                if layer_idx != len(self.weights) - 1:
+                    js_lines.append(f"    return relu({terms} + {b_term});\n}}\n\n")
+                else:
+                    js_lines.append(f"    return {terms} + {b_term};\n}}\n\n")
+                neuron_id += 1
+
+        neuron_counter = 0
+        for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)):
+            out_dim = len(W[0])
+            js_lines.append(f"function layer_{layer_idx}(inputs) {{\n")
+            inner = ", ".join([f"neuron_{neuron_counter + j}(inputs)" for j in range(out_dim)])
+            js_lines.append(f"    return [{inner}];\n}}\n\n")
+            neuron_counter += out_dim
+
+        js_lines.append("function predict(inputs) {\n")
+        js_lines.append("    let a = inputs;\n")
+        for i in range(len(self.weights)):
+            js_lines.append(f"    a = layer_{i}(a);\n")
+        js_lines.append("    return softmax(a);\n}\n\n")
+
+        js_lines.append(f"const vocab = {self.vocab};\n")
+        js_lines.append("const word_to_idx = {};\n")
+        js_lines.append("vocab.forEach((w, i) => { word_to_idx[w] = i; });\n")
+        js_lines.append(f"const context_window = {self.context_window};\n\n")
+        
+        # Add interactive browser logic
+        js_lines.append("""
+function completeText(inputText) {
+    const words = inputText.trim().split(/\\s+/).filter(w => w.length > 0);
+    let generatedWords = [...words];
+
+    for (let i = 0; i < 20; i++) {
+        const context = generatedWords.slice(-context_window);
+        const x = Array(vocab.length).fill(0.0);
+        
+        context.forEach(word => {
+            if (word in word_to_idx) {
+                x[word_to_idx[word]] = 1.0;
+            }
+        });
+
+        const out = predict(x);
+        const maxProb = Math.max(...out);
+        const idx = out.indexOf(maxProb);
+        const nextWord = vocab[idx];
+        
+        if (nextWord === '<|endoftext|>') {
+            break;
+        }
+        generatedWords.push(nextWord);
+    }
+    return generatedWords.join(' ');
+}
+
+document.addEventListener('DOMContentLoaded', () => {
+    const generateButton = document.getElementById('generateButton');
+    const userInput = document.getElementById('userInput');
+    const outputText = document.getElementById('outputText');
+
+    generateButton.addEventListener('click', () => {
+        const text = userInput.value;
+        if (text) {
+            const result = completeText(text);
+            outputText.textContent = result;
+        }
+    });
+});
+""")
+        with open(js_filename, "w") as f:
+            f.writelines(js_lines)
+
+        # --- Create HTML File ---
+        html_lines = [
+            '<!DOCTYPE html>\n',
+            '<html lang="en">\n',
+            '<head>\n',
+            '    <meta charset="UTF-8">\n',
+            '    <meta name="viewport" content="width=device-width, initial-scale=1.0">\n',
+            '    <title>Neural Network Text Completion</title>\n',
+            f'    <script src="{os.path.basename(js_filename)}"></script>\n',
+            '    <style>\n',
+            '        body { font-family: sans-serif; margin: 2em; background: #f0f0f0; }\n',
+            '        .container { max-width: 600px; margin: auto; background: white; padding: 2em; border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }\n',
+            '        h1 { text-align: center; color: #333; }\n',
+            '        textarea { width: 95%; height: 80px; padding: 10px; margin-bottom: 1em; border-radius: 4px; border: 1px solid #ccc; font-size: 1em; }\n',
+            '        button { display: block; width: 100%; padding: 10px; background: #007bff; color: white; border: none; border-radius: 4px; font-size: 1.1em; cursor: pointer; }\n',
+            '        button:hover { background: #0056b3; }\n',
+            '        #outputText { margin-top: 1.5em; padding: 1em; background: #e9ecef; border-radius: 4px; min-height: 40px; white-space: pre-wrap; word-wrap: break-word; }\n',
+            '    </style>\n',
+            '</head>\n',
+            '<body>\n',
+            '    <div class="container">\n',
+            '        <h1>Text Completion Model</h1>\n',
+            '        <textarea id="userInput" placeholder="Enter starting text... (e.g., user: hi)"></textarea>\n',
+            '        <button id="generateButton">Generate</button>\n',
+            '        <h3>Output:</h3>\n',
+            '        <div id="outputText"></div>\n',
+            '    </div>\n',
+            '</body>\n',
+            '</html>\n'
+        ]
+        with open(html_filename, "w") as f:
+            f.writelines(html_lines)
+            
+        print(f"Exported browser-runnable model to {js_filename} and {html_filename}")
+
+    @staticmethod
+    def load_network(filename):
+        ns = {"__name__": "__loaded_model__"}
+        with open(filename, "r") as f:
+            code = f.read()
+        exec(code, ns)
+        class ModelWrapper:
+            def __init__(self, ns):
+                self.ns = ns
+                self.vocab = ns.get("vocab", [])
+                self.word_to_idx = ns.get("word_to_idx", {})
+                self.context_window = ns.get("context_window", 5)
+
+            def complete(self, input_text, max_new_words=20):
+                words = [w.strip() for w in input_text.strip().split(' ') if w.strip()]
+                generated = words[:]
+                for _ in range(max_new_words):
+                    context = generated[-self.context_window:]
+                    x = [0.0] * len(self.vocab)
+                    for word in context:
+                        if word in self.word_to_idx:
+                            x[self.word_to_idx[word]] = 1.0
+                    
+                    out = self.ns["predict"](x)
+                    idx = out.index(max(out))
+                    next_word = self.vocab[idx]
+                    
+                    if next_word == '<|endoftext|>':
+                        break
+                    generated.append(next_word)
+                return ' '.join(generated)
+
+        return ModelWrapper(ns)
+
+
+if __name__ == "__main__":
+    sample_text = """
+user: hi
+ai: Hello! How can I help you today?
+<|endoftext|>
+user: hi
+ai: Hi! What can I do for you today?
+<|endoftext|>
+user: hello
+ai: Hello! How can I help you today?
+<|endoftext|>
+user: hey
+ai: Hi! What can I do for you today?
+<|endoftext|>
+user: How's your day going?
+ai: It's been great! Thanks for asking! How about yours?
+<|endoftext|>
+user: What's new with you?
+ai: Not much, just here and ready to help! What's new with you?
+<|endoftext|>
+user: What can you do?
+ai: I can help you with a variety of tasks. What's on your mind?
+<|endoftext|>
+user: Tell me a joke.
+ai: Why did the scarecrow win an award? Because he was outstanding in his field!
+<|endoftext|>
+"""
+    nn = NeuralNetwork(context_window=CONTEXT_WINDOW, seed=42)
+    nn.train(training_data=sample_text, lr=LR, epochs=EPOCHS, verbose_every=50)
+    
+    # Export both Python and JavaScript versions
+    nn.export_to_python("exported_model.py")
+    nn.export_to_js("web_model") # This will create web_model.js and web_model.html
+
+    model = NeuralNetwork.load_network("exported_model.py")
+    print("\n--- Testing loaded Python model ---")
+    print(f"Vocabulary size: {len(model.vocab)}")
+    
+    test_inputs = ["user: hi", "user: What's new", "ai: It's been"]
+    for test_input in test_inputs:
+        completion = model.complete(test_input, max_new_words=10)
+        print(f"Input: '{test_input}'\nOutput: '{completion}'\n")
+
+    print("\nTo test the JavaScript model, open 'web_model.html' in your browser.")